{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9dbd0268",
   "metadata": {},
   "source": [
    "自然语言处理\n",
    "\n",
    "1. 自然语言的张量表示, one-hot,word-bag,\n",
    "2. 词嵌入: skip-gram, bow\n",
    "\n",
    "\n",
    "定风波·莫听穿林打叶声\n",
    "\n",
    "三月七日，沙湖道中遇雨，雨具先去，同行皆狼狈，余独不觉。已而遂晴，故作此(词)。\n",
    "\n",
    "莫听穿林打叶声，何妨吟啸且徐行。\n",
    "\n",
    "竹杖芒鞋轻胜马，谁怕？一蓑烟雨任平生。\n",
    "\n",
    "料峭春风吹酒醒，微冷，山头斜照却相迎。\n",
    "\n",
    "回首向来萧瑟处，归去，也无风雨也无晴。\n",
    "\n",
    "\n",
    "## 对于自然语言，如何用向量来表示？\n",
    "\n",
    "### 第一步，对于语料库的文本拆分统计，得到一个词典，为每个字分配一个位置编号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3e5140b0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'打', '徐', '先', '觉', '雨', '同', '作', '且', '(', '谁', '波', '日', '蓑', '狈', '首', '料', '独', '故', '遇', '也', ' ', '啸', '三', '一', '斜', '却', '归', '穿', '照', '风', '平', '具', '鞋', '相', '定', '春', '胜', '杖', '月', '处', ')', '叶', '无', '晴', '芒', '沙', '回', '此', '吟', '·', '瑟', '酒', '醒', '余', '妨', '莫', '，', '遂', '怕', '？', '任', '湖', '竹', '林', '轻', '而', '听', '。', '生', '峭', '道', '不', '山', '向', '微', '行', '去', '烟', '迎', '皆', '头', '已', '七', '马', '来', '中', '何', '词', '冷', '吹', '狼', '萧', '声'}\n",
      "93 {'打': 0, '谁': 1, '波': 2, '日': 3, '首': 4, '料': 5, ' ': 6, '啸': 7, '三': 8, '一': 9, '斜': 10, '穿': 11, '照': 12, '具': 13, '鞋': 14, '相': 15, '定': 16, '春': 17, '胜': 18, ')': 19, '叶': 20, '晴': 21, '·': 22, '瑟': 23, '余': 24, '莫': 25, '，': 26, '遂': 27, '怕': 28, '？': 29, '竹': 30, '而': 31, '生': 32, '道': 33, '山': 34, '向': 35, '微': 36, '去': 37, '烟': 38, '七': 39, '中': 40, '词': 41, '冷': 42, '吹': 43, '狼': 44, '萧': 45, '徐': 46, '先': 47, '觉': 48, '雨': 49, '同': 50, '作': 51, '且': 52, '(': 53, '蓑': 54, '狈': 55, '独': 56, '故': 57, '遇': 58, '也': 59, '却': 60, '归': 61, '风': 62, '平': 63, '杖': 64, '月': 65, '处': 66, '无': 67, '芒': 68, '回': 69, '此': 70, '吟': 71, '酒': 72, '醒': 73, '妨': 74, '任': 75, '湖': 76, '林': 77, '轻': 78, '听': 79, '。': 80, '峭': 81, '不': 82, '行': 83, '迎': 84, '皆': 85, '头': 86, '已': 87, '马': 88, '来': 89, '何': 90, '沙': 91, '声': 92}\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "txt = \"定风波·莫听穿林打叶声 三月七日，沙湖道中遇雨，雨具先去，同行皆狼狈，余独不觉。已而遂晴，故作此(词)。莫听穿林打叶声，何妨吟啸且徐行。竹杖芒鞋轻胜马，谁怕？一蓑烟雨任平生。料峭春风吹酒醒，微冷，山头斜照却相迎。回首向来萧瑟处，归去，也无风雨也无晴。\"\n",
    "tokens = set([token for token in txt]) \n",
    "\n",
    "print(set(tokens))\n",
    "\n",
    "vocabs = {}\n",
    "for idx,token in enumerate(tokens):\n",
    "    vocabs[token]=idx\n",
    "\n",
    "vocabs_len = len(vocabs)\n",
    "print(vocabs_len,vocabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cca8b19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pytorch 针对自然语言处理的一些封装\n",
    "from torchtext.vocab import vocab\n",
    "from collections import Counter, OrderedDict\n",
    "\n",
    "counter = Counter([token for token in txt])\n",
    "sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[1], reverse=True)\n",
    "ordered_dict = OrderedDict(sorted_by_freq_tuples)\n",
    "\n",
    "\n",
    "v1 = vocab(ordered_dict)\n",
    "print(v1['山'])\n",
    "print(v1.lookup_indices(['山','定']))\n",
    "\n",
    "# torch.txt 0.12支持\n",
    "unk_token = '<unk>'\n",
    "default_index = -1\n",
    "# v2 = vocab(OrderedDict([(token, 1) for token in txt]), specials=[unk_token])\n",
    "# v2.set_default_index(default_index)\n",
    "# print(v2['<unk>']) #prints 0\n",
    "# print(v2['out of vocab']) #prints -1\n",
    "# #make default index same as index of unk_token\n",
    "# v2.set_default_index(v2[unk_token])\n",
    "# v2['out of vocab'] is v2[unk_token] #prints True\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c49e1d0",
   "metadata": {},
   "source": [
    "### 第二步，词向量表达"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bbcdac15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "(7, 93)\n",
      "[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "# 1. one-hot 方式，一个词典大小维度的向量，例如上面的词典总计有93维，\"晴\"在词典中位置=39，\"晴\"可以表示为39位置是1，其他位置都是0的向量\n",
    "# 缺点：稀疏矩阵，参与计算时很浪费\n",
    "\n",
    "# 单个字，映射为一个向量\n",
    "def convert_word_to_vector(word):\n",
    "    word_idx = vocabs[word]\n",
    "#     print(word_idx)\n",
    "    vector = np.zeros(vocabs_len)\n",
    "    vector[word_idx]=1\n",
    "#     print(vector)\n",
    "    return vector\n",
    "    \n",
    "v = convert_word_to_vector(word = \"晴\")\n",
    "print(v)\n",
    "\n",
    "# 输入是一句话，映射为多个向量构成的矩阵，最后会产出一个n*m 的矩阵，m为词典大小，n为输入句子的长度\n",
    "input = \"莫听穿林打叶声\"\n",
    "vector_li = []\n",
    "for token in input:\n",
    "#     print(token)\n",
    "    v = convert_word_to_vector(token)\n",
    "    vector_li.append(v)\n",
    "\n",
    "out = np.array(vector_li)\n",
    "print(out.shape)\n",
    "print(out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "41e01a42",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0.\n",
      " 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 1. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]\n"
     ]
    }
   ],
   "source": [
    "# 2. 词袋法(Bag of Words), 对于输入一个句子来说，产出的不是一个矩阵，而是一个向量，维度为词典大小，其他位置0,出现的字的位置为1。\n",
    "# 除了将出现字的位置设置为1外，还可以设置为该字在整个句子中出现的次数，或者tf-idf值\n",
    "# idf(逆文档频率) = log(语料库中文档总数/包含该词的文档数+1)\n",
    "# tf(词频) = 某个词在文章中的出现次数/文章总词数\n",
    "# tf-idf = tf * idf \n",
    "# 缺点：输词的位置信息丢了。\n",
    "input = \"莫听穿林打叶声\"\n",
    "\n",
    "vector = np.zeros(vocabs_len)\n",
    "for token in input:\n",
    "    word_idx = vocabs[token]\n",
    "    vector[word_idx]=1\n",
    "\n",
    "print(vector)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "932a5a6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(93, 5)\n",
      "[25, 79, 11, 77, 0, 20, 92]\n",
      "[[0.60639321 0.0191932  0.30157482 0.66017354 0.29007761]\n",
      " [0.95898272 0.35536885 0.35670689 0.0163285  0.18523233]\n",
      " [0.16130952 0.65310833 0.2532916  0.46631077 0.24442559]\n",
      " [0.90884372 0.81552382 0.15941446 0.62889844 0.39843426]\n",
      " [0.5488135  0.71518937 0.60276338 0.54488318 0.4236548 ]\n",
      " [0.67781654 0.27000797 0.73519402 0.96218855 0.24875314]\n",
      " [0.2724369  0.3790569  0.37429618 0.74878826 0.23780724]]\n"
     ]
    }
   ],
   "source": [
    "# 3. 词嵌入 Word embedding\n",
    "# 设定一个词为一个128维的向量，先随机初始化这个向量，通过某种方式调整这个向量大小，经过训练后会行程一个 n*m, n为词典大小，m为每个词向量大小。\n",
    "vector_size = 5\n",
    "WordEmbedding = np.random.rand(vocabs_len,vector_size)\n",
    "print(WordEmbedding.shape)\n",
    "\n",
    "txt_input = \"莫听穿林打叶声\"\n",
    "words_idx = [vocabs[token] for token in txt_input]\n",
    "print(words_idx)\n",
    "print(WordEmbedding[words_idx,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1aaf653f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[-0.0263,  0.4056, -0.5449,  0.3652, -1.1004],\n",
      "         [-0.4647,  1.6371,  1.5121,  0.1880,  0.6070],\n",
      "         [-0.5138, -0.9206, -1.1940,  0.2056,  0.8584],\n",
      "         [-2.0510, -0.6589, -0.8776,  0.5585, -0.0444],\n",
      "         [-0.2092, -0.7802,  1.3900, -0.9586,  0.2622],\n",
      "         [ 1.5567, -1.1102, -1.2972,  0.0290, -0.0998],\n",
      "         [-0.1247,  0.2919, -1.3336,  0.3391, -0.9337]]],\n",
      "       grad_fn=<EmbeddingBackward>)\n"
     ]
    }
   ],
   "source": [
    "# 基于pytorch的嵌入\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "embedding = nn.Embedding(vocabs_len,vector_size)\n",
    "words_idx_t = torch.LongTensor([words_idx])\n",
    "print(embedding(words_idx_t))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ceb6026",
   "metadata": {},
   "source": [
    "### 如何调整Word embedding矩阵中每个词向量的值\n",
    "\n",
    "自然语言模型中，哪些词是经常一起出现的，哪些不是？ 给定左右出现的词后，它们中间的词会是哪些？CBOW(Continuous Bag-of-Words Model)；或者给定一个词或者一个字，它左右经常出现的词是哪些？Skip-Gram Continuous skip-gram Model.\n",
    "\n",
    "* CBOW: 优点:训练稳定，容易收敛；缺点:生僻词效果不佳\n",
    "* Skip-gram: 优点：对生僻字不敏感， 缺点：不太稳定？\n",
    "\n",
    "接下来使用Skip-gram尝试训练词嵌入\n",
    "1. 准备训练数据\n",
    "设置一个滑动窗口，一般为5，演示方便设置为3，从语料库开始位置，选取滑动窗口大小的字，中间的为输入1，两边的为输入2\n",
    "2. 搭建模型\n",
    "3. 训练词嵌入\n",
    "4. 词嵌入的一些特点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f3b277a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in_out: 波 定\n",
      "in_out: 波 风\n",
      "in_out: 波 ·\n",
      "in_out: 波 莫\n",
      "in_out: · 风\n",
      "in_out: · 波\n",
      "in_out: · 莫\n",
      "in_out: · 听\n",
      "in_out: 莫 波\n",
      "in_out: 莫 ·\n",
      "in_out: 莫 听\n",
      "in_out: 莫 穿\n",
      "in_out: 听 ·\n",
      "in_out: 听 莫\n",
      "in_out: 听 穿\n",
      "in_out: 听 林\n",
      "in_out: 穿 莫\n",
      "in_out: 穿 听\n",
      "in_out: 穿 林\n",
      "in_out: 穿 打\n",
      "in_out: 林 听\n",
      "in_out: 林 穿\n",
      "in_out: 林 打\n",
      "in_out: 林 叶\n",
      "in_out: 打 穿\n",
      "in_out: 打 林\n",
      "in_out: 打 叶\n",
      "in_out: 打 声\n"
     ]
    }
   ],
   "source": [
    "#思路1, 输入是中间词，输出是上下文词，过模型，softmax后预测词表中每个词的概率值，缺点：由于实际中词表非长大，计算量大，浪费资源\n",
    "slide_windows_cnt = 5\n",
    "for i in range(len(txt)):\n",
    "    windows_token = txt[i:i+slide_windows_cnt]\n",
    "    middle_idx = (slide_windows_cnt-1)//2\n",
    "    middle = windows_token[middle_idx]\n",
    "    for j in range(slide_windows_cnt):\n",
    "        if j != middle_idx:\n",
    "            print(\"in_out:\",windows_token[middle_idx],windows_token[j])\n",
    "#     print(middle,windows_token)\n",
    "    if i>5:\n",
    "        break "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4e463f59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['无', '芒', '此', '一']\n",
      "(array(1., dtype=float32), 2, 16)\n",
      "label batch shape: torch.Size([10]) tensor([0., 0., 0., 0., 1., 0., 0., 1., 0., 1.])\n",
      "in_idx batch shape: torch.Size([10]) tensor([39, 62, 32, 32, 36, 26, 88, 12, 67, 77])\n",
      "cxt_idx batch shape: torch.Size([10]) tensor([69, 59, 49, 18, 26, 15,  9, 15, 73, 11])\n"
     ]
    }
   ],
   "source": [
    "# 思路2, 模型接受2个输入，中间词+真实的上下文词(或从词典里随机抽些词)，\n",
    "# 输出：中间词+真实上下文词输出1，中间词+负采样输出0，将多分类问题转化为一个二分类问题\n",
    "def Negative_sampling(tokens):\n",
    "    return random.choices(list(tokens),k=slide_windows_cnt-1) #random.randint(0,tokens)\n",
    "\n",
    "def load_dataset(txt,tokens):\n",
    "    dataset = []\n",
    "    slide_windows_cnt = 5\n",
    "    for i in range(len(txt)):\n",
    "        windows_token = txt[i:i+slide_windows_cnt]\n",
    "        middle_idx = (slide_windows_cnt-1)//2 \n",
    "        if middle_idx>len(windows_token)-1:\n",
    "            break\n",
    "        middle = windows_token[middle_idx]\n",
    "        for j in range(len(windows_token)):\n",
    "            if j != middle_idx:\n",
    "#                 print(j,middle_idx)\n",
    "                dataset.append([1,windows_token[middle_idx],windows_token[j]])\n",
    "#                 print(\"\\t\".join([\"1\",windows_token[middle_idx],windows_token[j]]))\n",
    "        for token in Negative_sampling(tokens):\n",
    "            dataset.append([0,windows_token[middle_idx],token])\n",
    "#             print(\"\\t\".join([\"0\",windows_token[middle_idx],token])\n",
    "    return dataset \n",
    "\n",
    "load_dataset(txt,tokens)\n",
    "\n",
    "# 实际生成 label,middel,context时，需要考虑词在语料库中出现的次数，对于出现次数较多的词，需要减少他们在生成数据中的出现次数。\n",
    "print(Negative_sampling(tokens))\n",
    "\n",
    "\n",
    "\n",
    "# 使用pytorch, Dataset,DataLoader的方式实现训练数据的加载\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "class LanguageDataset(Dataset):\n",
    "    def __init__(self):\n",
    "      self.dataset = load_dataset(txt,tokens)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, idx): \n",
    "        row = self.dataset[idx] \n",
    "        label = row[0]\n",
    "        in_idx = vocabs[row[1]]\n",
    "        cxt_idx = vocabs[row[2]]\n",
    "#         print(label,in_idx,cxt_idx)\n",
    "        \n",
    "        return np.array(label,dtype=np.float32),in_idx,cxt_idx\n",
    "\n",
    "data = LanguageDataset()\n",
    "print(data[0])\n",
    "\n",
    "train_dataloader = DataLoader(data, batch_size=10, shuffle=True)\n",
    "label, in_idx,cxt_idx = next(iter(train_dataloader))\n",
    "print(f\"label batch shape: {label.size()}\",label)\n",
    "print(f\"in_idx batch shape: {in_idx.size()}\",in_idx)\n",
    "print(f\"cxt_idx batch shape: {cxt_idx.size()}\",cxt_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "87748001",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out: tensor([[[ 0.1401,  0.0610,  1.0358, -0.3441, -1.0490]]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([1, 1]) tensor([[0.4610]], grad_fn=<SigmoidBackward>)\n",
      "tensor([0., 0., 1., 0., 0., 1., 1., 0., 0., 1.]) tensor([ 0, 72, 83,  8, 80, 79, 80, 26, 59, 45]) tensor([89, 74, 80, 31, 78, 77, 21, 43, 92, 89])\n",
      "out: tensor([[-0.0751,  1.0648,  0.0086,  0.9577,  0.4810],\n",
      "        [-0.6918, -0.0039,  1.9472, -0.7381,  2.2029],\n",
      "        [-1.5257, -0.0879, -0.3471,  0.9209,  0.2662],\n",
      "        [ 0.8744,  0.1277, -1.2394,  0.2109, -1.4333],\n",
      "        [-1.5130,  0.0870,  0.9916,  0.9618, -0.1716],\n",
      "        [ 0.0224, -1.7983,  0.8457, -0.5641, -0.3229],\n",
      "        [-0.6765, -0.5366,  0.7538,  0.6974, -0.2330],\n",
      "        [-0.2077, -1.2751,  0.8477,  0.0059, -0.1134],\n",
      "        [-2.5775,  0.6961, -0.6527,  1.1731,  0.2835],\n",
      "        [-0.0856, -0.5757, -0.1493, -0.2413, -1.8482]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9196, 0.9380, 0.3157, 0.1885, 0.5880, 0.1398, 0.5012, 0.3224, 0.2540,\n",
      "        0.0522], grad_fn=<SigmoidBackward>)\n",
      "tensor([0.9196, 0.9380, 0.3157, 0.1885, 0.5880, 0.1398, 0.5012, 0.3224, 0.2540,\n",
      "        0.0522], grad_fn=<SigmoidBackward>)\n"
     ]
    }
   ],
   "source": [
    "# Skip-gram model\n",
    "class SkipgramModel(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_size):\n",
    "        super(SkipgramModel, self).__init__()\n",
    "        self.emb_in = nn.Embedding(vocab_size, embedding_size)\n",
    "        self.emb_cxt = nn.Embedding(vocab_size, embedding_size) \n",
    "\n",
    "    def forward(self, inToken,cxtToken): \n",
    "        in_tensor = self.emb_in(inToken)\n",
    "#         print(in_tensor.shape,in_tensor)\n",
    "        cxt_tensor = self.emb_cxt(cxtToken)\n",
    "#         print(cxt_tensor.shape,cxt_tensor) \n",
    "#         tmp = in_tensor * cxt_tensor \n",
    "        out = torch.mul(in_tensor, cxt_tensor)\n",
    "        print(\"out:\",out)\n",
    "        out = torch.sum(out, dim=-1)\n",
    "        out = torch.sigmoid(out)\n",
    "        print(out.shape,out)\n",
    "        return out\n",
    "\n",
    "model = SkipgramModel(vocabs_len,vector_size)\n",
    "\n",
    "#单个例子test\n",
    "model(torch.tensor([[1]]),torch.tensor([[2]]))\n",
    "\n",
    "#多个测试\n",
    "label, in_idx,cxt_idx = next(iter(train_dataloader))\n",
    "print(label, in_idx,cxt_idx)\n",
    "pred = model(in_idx,cxt_idx)\n",
    "print(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1f6dc093",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out: tensor([[ 5.1338e-01, -3.2567e-02,  2.2558e-01, -1.1504e-01, -1.3168e+00],\n",
      "        [ 3.2692e-03, -3.6338e+00,  1.7465e-01, -7.1994e-01,  1.2621e+00],\n",
      "        [ 6.7210e-01, -3.1703e-01, -2.0234e+00,  3.5504e-01,  1.4407e+00],\n",
      "        [ 7.7532e-01,  2.0064e-01, -1.1363e+00, -3.3789e-01,  6.0580e-01],\n",
      "        [ 4.5659e-01, -1.0578e+00, -3.6325e-01,  1.5888e-01,  1.2875e-01],\n",
      "        [ 2.9871e-03,  5.5018e-02, -7.7735e-01,  3.9210e-01,  1.4635e+00],\n",
      "        [-3.1587e-01, -3.9140e-01,  7.9908e-01, -1.5911e-01,  1.1052e-01],\n",
      "        [-4.6835e-02, -3.5396e-03, -8.7775e-01, -3.0353e-02, -1.4621e-01],\n",
      "        [ 6.8691e-01, -1.6944e+00,  2.6739e-01, -6.6844e-02,  1.1297e+00],\n",
      "        [ 1.2454e-01,  4.4316e-01, -3.2308e-01, -3.3373e-01, -9.2537e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3262, 0.0515, 0.5318, 0.5269, 0.3370, 0.7570, 0.5108, 0.2489, 0.5800,\n",
      "        0.2661], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.720184  [    0/  981]\n",
      "out: tensor([[ 1.0991e-01, -3.1363e-02, -4.4559e-01, -5.4421e-01, -1.6615e+00],\n",
      "        [ 3.2474e-03, -3.8446e+00,  3.7549e-02,  1.6381e-01, -1.9710e+00],\n",
      "        [-1.2814e+00, -2.9799e-01,  4.8382e-01, -2.4373e-01,  6.2642e-01],\n",
      "        [-8.9070e-03, -2.4723e-02,  3.0520e-01,  5.5543e-01,  2.1790e-01],\n",
      "        [-9.9834e-02,  1.7075e+00,  2.0512e+00, -5.1613e-01, -1.0452e-01],\n",
      "        [-3.0514e-02,  4.2311e-01,  9.5301e-01, -1.2925e-01,  3.2081e-01],\n",
      "        [ 7.8696e-02,  6.4529e-03,  6.5827e-02,  3.3429e-01, -5.8029e-02],\n",
      "        [-7.4263e-01, -1.7035e+00,  7.7938e-01,  2.6726e+00, -1.3520e-01],\n",
      "        [ 7.7168e-04,  7.3999e-01,  1.0195e-01,  4.8883e-01,  2.7863e-01],\n",
      "        [ 9.9453e-01,  1.0374e-02, -8.8118e-02,  3.3534e-01, -4.0834e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0709, 0.0036, 0.3290, 0.7398, 0.9543, 0.8231, 0.6052, 0.7049, 0.8334,\n",
      "        0.6993], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.9928, -1.5262,  0.2461, -0.4754,  2.1498],\n",
      "        [-0.0224, -1.0166,  0.1633, -0.0199, -0.2547],\n",
      "        [ 0.6489,  0.2521,  0.0165,  1.1307,  0.0168],\n",
      "        [ 0.7965,  0.0064, -0.0762, -0.2429, -0.9181],\n",
      "        [-0.4873,  0.8678, -0.7271, -0.2077, -1.7283],\n",
      "        [ 0.1030,  0.1909, -0.2726, -0.8810,  1.0329],\n",
      "        [ 0.0224, -1.7983,  0.8457, -0.5641, -0.3229],\n",
      "        [ 0.0342, -0.0131, -0.8018, -0.0499,  0.2064],\n",
      "        [-0.5988,  2.2657, -0.0954,  0.0928, -0.1330],\n",
      "        [-1.2106, -2.1897, -1.0939,  0.3332,  0.2364]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8001, 0.2404, 0.8875, 0.3931, 0.0926, 0.5432, 0.1398, 0.3488, 0.8222,\n",
      "        0.0194], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.4009e-01,  6.0982e-02,  1.0358e+00, -3.4410e-01, -1.0490e+00],\n",
      "        [-3.9954e-02,  2.1993e-03,  8.5899e-01,  5.1916e-01, -1.7833e-01],\n",
      "        [-3.0287e+00,  2.2116e-02, -1.2934e+00, -1.3448e+00,  2.2012e-02],\n",
      "        [ 1.6093e+00,  7.3580e-03,  2.1179e-01, -3.6858e-01, -9.3237e-02],\n",
      "        [ 3.3658e-02,  3.8247e-01,  7.5649e-02, -1.4842e+00,  7.1186e-01],\n",
      "        [ 7.1353e-02, -4.0965e-01,  8.8425e-01,  6.8763e-01,  6.9077e-01],\n",
      "        [-6.8947e-01, -2.0350e+00,  6.3502e-02,  2.1850e-01,  4.2703e-01],\n",
      "        [-6.2187e-01, -7.8368e-02,  1.2268e-01,  1.9012e+00,  1.0094e+00],\n",
      "        [ 3.9107e-01,  3.5405e-03,  5.4798e-01, -2.5290e-01,  3.2883e+00],\n",
      "        [-3.0038e-01,  4.9792e-02, -1.4155e+00,  1.1064e-01,  1.3420e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.4610, 0.7617, 0.0036, 0.7968, 0.4303, 0.8726, 0.1176, 0.9116, 0.9816,\n",
      "        0.1945], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-2.3190e+00, -1.7320e-01,  5.6766e-01, -6.3456e-01, -9.6425e-01],\n",
      "        [-1.3420e-01,  8.4243e-01, -2.5217e-01, -6.3595e-01, -1.0696e+00],\n",
      "        [ 2.3288e-01,  6.6707e-03, -6.5901e-02,  5.7453e-01, -5.6079e-02],\n",
      "        [-3.8547e-01, -1.4107e-01,  3.1976e-01,  1.2478e-01,  7.9877e-01],\n",
      "        [ 1.3476e+00,  3.1863e-01, -5.1130e-01,  5.6879e-03,  8.4043e-02],\n",
      "        [-7.7855e-02, -1.4468e-02, -6.4356e-01,  3.3164e-01, -7.7047e-01],\n",
      "        [ 2.5857e-01, -8.9745e-03, -2.9716e-01, -5.4181e-02, -1.3515e-01],\n",
      "        [-8.0362e-01,  1.4663e+00, -2.5070e-01,  6.3943e-02,  6.9912e-01],\n",
      "        [-6.8414e-02,  2.0671e+00,  1.9675e-01,  2.8797e-01, -2.1534e+00],\n",
      "        [ 1.6038e-01, -5.8369e-01, -2.2112e-01,  1.3999e-04, -1.0014e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0287, 0.2228, 0.6664, 0.6719, 0.7764, 0.2360, 0.4410, 0.7641, 0.5818,\n",
      "        0.1617], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.5308,  0.5478,  0.8995, -0.2013,  0.3326],\n",
      "        [ 0.0959,  0.5542,  0.4532,  0.2258, -0.0726],\n",
      "        [ 0.1570, -1.3720, -0.4037,  0.4144,  0.0474],\n",
      "        [ 0.6987, -0.1362, -1.4273,  0.0480, -0.0178],\n",
      "        [-0.3301,  0.5769, -1.1266,  0.0195, -0.1270],\n",
      "        [ 0.8672,  0.1841,  1.0804, -0.6082,  0.1656],\n",
      "        [ 0.1950,  0.0091,  0.3215,  0.6246,  0.2461],\n",
      "        [ 0.4454,  0.4820,  0.7684,  0.2594, -0.2202],\n",
      "        [ 0.3252,  0.3529, -0.4563,  0.0543,  0.0710],\n",
      "        [ 1.5628,  0.1674,  3.2126, -0.3909, -1.8372]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8918, 0.7784, 0.2392, 0.3027, 0.2715, 0.8441, 0.8016, 0.8501, 0.5859,\n",
      "        0.9379], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.531329  [   50/  981]\n",
      "out: tensor([[-0.2099,  0.2620, -0.7756,  0.5721,  1.0510],\n",
      "        [-0.0377, -0.2285,  1.2258, -0.7563,  0.3639],\n",
      "        [ 0.3773, -0.1052, -0.0210,  1.2340, -0.0758],\n",
      "        [ 0.7643, -0.5237, -1.0605, -1.4569,  0.1560],\n",
      "        [-0.3949,  0.5300,  0.4874, -0.2113, -0.1945],\n",
      "        [-0.3059, -1.7919, -0.6472, -2.1674, -0.3270],\n",
      "        [ 0.1309, -0.3133,  0.1499,  0.0259, -0.0652],\n",
      "        [-0.8426, -0.6330,  0.3111, -0.5035, -1.0545],\n",
      "        [ 1.0688, -0.0219,  0.0084, -0.3431, -1.8203],\n",
      "        [-0.0148, -0.0339,  0.1124,  0.0413,  1.2578]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7109, 0.6381, 0.8036, 0.1071, 0.5540, 0.0053, 0.4820, 0.0617, 0.2482,\n",
      "        0.7962], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.1801, -2.3444, -0.6239, -0.3838,  0.0602],\n",
      "        [ 0.3952, -0.2622, -2.8934, -1.4265, -0.1282],\n",
      "        [-0.0366,  0.4101, -1.0863, -1.4209,  0.1731],\n",
      "        [-0.1004,  1.7548, -0.0207,  2.0955,  0.0197],\n",
      "        [ 1.2642, -1.2546, -0.5109, -0.5039, -0.3096],\n",
      "        [-0.0088, -0.0303, -0.2421, -0.0868,  1.0304],\n",
      "        [-0.0517,  0.1671, -1.0009,  0.0502, -1.0688],\n",
      "        [ 0.8825, -0.1733, -0.4482,  0.1040,  0.0128],\n",
      "        [-0.1701,  1.9460,  0.7592, -2.6420,  1.0235],\n",
      "        [ 1.5340, -1.3884,  0.6313,  0.2128, -1.6455]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0426, 0.0132, 0.1234, 0.9770, 0.2117, 0.6598, 0.1296, 0.5934, 0.7143,\n",
      "        0.3417], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-1.0104e+00, -9.7158e-01, -1.6949e-01, -1.8986e+00, -1.8997e-02],\n",
      "        [-4.2891e-01,  2.7292e+00, -2.7740e-02,  1.2231e+00,  1.4359e-01],\n",
      "        [ 1.0135e+00,  2.5749e-01, -2.2071e+00,  2.1273e+00, -3.7655e-02],\n",
      "        [-5.2833e-04,  7.1572e-01,  1.8936e+00, -1.5971e+00, -6.5697e-01],\n",
      "        [ 7.7070e-01,  1.6948e-02,  4.4076e-01, -3.0459e-01, -3.8999e-01],\n",
      "        [ 1.1044e+00,  4.9109e-01, -1.7240e+00,  2.6829e-01, -2.2613e+00],\n",
      "        [ 4.9474e-01, -1.2082e+00,  3.3735e-01, -1.9307e-01, -1.1368e+00],\n",
      "        [ 4.6059e-01, -2.8263e-02,  1.4740e-01, -1.1005e+00, -2.0091e+00],\n",
      "        [ 1.0894e+00,  8.0084e-01,  5.3292e-01,  2.5575e-01,  1.6078e-01],\n",
      "        [-1.2190e+00, -1.5566e-01,  3.0977e-01,  1.7195e-01, -1.6369e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0168, 0.9744, 0.7602, 0.5878, 0.6304, 0.1070, 0.1537, 0.0738, 0.9448,\n",
      "        0.2580], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-3.8548e-01, -1.4107e-01,  3.1975e-01,  1.2478e-01,  7.9875e-01],\n",
      "        [ 7.2356e-02,  3.8277e-01,  7.5168e-01, -2.1727e+00, -5.8795e-01],\n",
      "        [-4.3276e-01,  1.8797e-01, -2.4636e-01, -1.8130e-01,  5.2302e-01],\n",
      "        [-5.4127e-01, -1.4225e+00, -6.5116e-01,  1.2771e-01, -1.2350e-01],\n",
      "        [ 4.0043e-01, -1.0362e+00, -7.4839e-01,  1.6833e+00, -1.8563e-01],\n",
      "        [-3.2179e-02,  8.6887e-01,  9.8655e-02, -1.5105e+00, -2.4910e-01],\n",
      "        [-3.7799e-01,  9.9501e-02,  2.5038e-01, -5.0269e-01,  2.7013e-02],\n",
      "        [-1.7247e-01,  4.0808e-02, -3.8515e-02,  1.7901e-01,  1.6241e+00],\n",
      "        [-1.0680e+00, -1.4318e+00, -2.7669e-01, -1.5586e+00,  7.8987e-01],\n",
      "        [ 8.6058e-01, -8.6564e-01,  1.7114e+00, -1.7493e-03,  1.2201e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.6719, 0.1745, 0.4627, 0.0685, 0.5283, 0.3049, 0.3767, 0.8366, 0.0281,\n",
      "        0.8614], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-1.6682e-01,  1.3727e+00,  6.0360e-02,  4.5261e-01,  1.5534e-03],\n",
      "        [-1.1581e-01, -9.6496e-02,  5.1435e-01, -1.0535e+00,  8.0242e-01],\n",
      "        [ 1.6538e-03, -1.3644e+00,  8.1793e-01,  1.9746e-01,  7.2488e-01],\n",
      "        [-5.6387e-01,  7.6487e-01,  5.1081e-01,  5.1463e-02, -8.4270e-01],\n",
      "        [-2.5347e+00,  1.5875e-01,  1.0632e+00, -3.0150e-01, -1.9914e-01],\n",
      "        [ 2.9599e-01,  2.1122e-01,  1.0868e-01,  5.7052e-01,  1.1990e+00],\n",
      "        [-2.3369e-01,  2.9295e-01,  1.3342e-01,  7.7344e-01, -3.9146e-02],\n",
      "        [-4.9801e-01, -1.5843e-02,  1.0793e-01, -2.2533e-02, -1.4993e+00],\n",
      "        [-9.6766e-02,  2.3517e-01, -8.2427e-01,  5.3104e-01,  5.0547e-01],\n",
      "        [-2.5479e-01,  1.0643e-01, -1.9481e-02,  5.5495e-01, -1.1896e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8482, 0.5127, 0.5933, 0.4802, 0.1402, 0.9157, 0.7165, 0.1270, 0.5868,\n",
      "        0.3095], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.110448  [  100/  981]\n",
      "out: tensor([[-0.2745, -0.2581, -5.6858, -0.1670,  0.1282],\n",
      "        [-0.3661,  0.0245, -0.0630,  0.1813, -1.5970],\n",
      "        [ 1.3751, -0.3005,  4.2505, -0.1224,  0.8667],\n",
      "        [ 0.5030, -0.0499,  1.2419,  0.5035, -1.1152],\n",
      "        [ 0.1801, -2.3443, -0.6239, -0.3838,  0.0601],\n",
      "        [-0.8293, -0.1261, -1.2753, -0.0831,  0.2995],\n",
      "        [-0.7116,  0.0315, -0.1546,  0.0677, -2.5103],\n",
      "        [ 0.0302,  2.2367,  1.0548,  1.2926, -1.4180],\n",
      "        [-0.0141,  0.2769, -0.1770, -0.1707,  1.2533],\n",
      "        [ 0.4143, -0.0431,  0.8817, -1.0744, -1.2115]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0019, 0.1394, 0.9977, 0.7471, 0.0426, 0.1177, 0.0364, 0.9607, 0.7629,\n",
      "        0.2625], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 2.9503e-01,  1.7442e+00,  4.5667e-01,  9.7790e-01, -1.2456e-01],\n",
      "        [ 3.6377e-01,  2.5821e-01,  1.3928e-02, -1.8114e-01, -4.1259e-01],\n",
      "        [-1.2629e-02, -9.2778e-01,  2.6562e-01,  2.3835e-01,  1.0446e-01],\n",
      "        [ 9.9542e-01, -4.5271e+00, -5.2546e-01, -5.8496e-02,  8.1939e-01],\n",
      "        [ 6.5950e-01, -9.4429e-02, -5.7249e-01,  6.2298e-01, -9.4299e-01],\n",
      "        [ 1.1345e-01, -1.0476e+00, -2.7619e+00, -4.2112e-01,  5.1328e-02],\n",
      "        [-2.4366e-03,  1.1275e+00, -3.8475e-01,  6.6100e-01, -8.9373e-01],\n",
      "        [ 5.5868e-01, -6.2750e-02,  4.5266e-01, -2.8521e-02,  1.7195e+00],\n",
      "        [ 4.3576e-01, -1.2591e-01,  6.9246e-02, -4.3246e-01, -1.5588e+00],\n",
      "        [ 1.8013e-01, -2.3443e+00, -6.2391e-01, -3.8381e-01,  6.0151e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9661, 0.5105, 0.4178, 0.0357, 0.4189, 0.0169, 0.6242, 0.9334, 0.1663,\n",
      "        0.0426], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.5408,  0.8528,  0.4846, -0.1771, -0.2629],\n",
      "        [ 0.2761, -0.0187, -0.1243, -0.2342, -0.0532],\n",
      "        [ 1.2518, -0.1735,  0.0942,  0.1464, -1.1199],\n",
      "        [ 1.2556,  0.1855, -0.8030, -0.2191, -0.0773],\n",
      "        [-0.5278,  0.7421, -0.8630,  0.5141, -1.3883],\n",
      "        [ 1.4391, -0.6778, -0.2267, -2.4826,  0.9361],\n",
      "        [ 0.0129, -0.4598,  0.3147, -0.1946,  0.4972],\n",
      "        [ 0.1600, -0.2596, -0.7474,  0.2041, -1.8426],\n",
      "        [ 0.0512, -0.8881,  0.1417,  0.0195,  0.2477],\n",
      "        [-0.9924,  0.2936, -0.0741,  0.0728,  0.0505]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.5882, 0.4615, 0.5496, 0.5846, 0.1790, 0.2666, 0.5425, 0.0769, 0.3946,\n",
      "        0.3431], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.3952, -0.1324, -0.0850,  0.0456,  1.3314],\n",
      "        [-0.6948, -0.4100,  0.4952, -1.2138, -0.0952],\n",
      "        [ 1.6512, -0.5822, -2.0804, -1.0697,  0.2021],\n",
      "        [-0.7330, -0.4044,  0.5937, -0.7050, -0.1033],\n",
      "        [-0.6353,  0.0901,  3.8205, -0.5309, -0.3070],\n",
      "        [-0.5727, -0.2199,  1.3419, -0.0812, -0.7750],\n",
      "        [-0.0617,  0.3005,  2.1517,  0.7482,  1.0024],\n",
      "        [-2.0739,  0.6409, -0.2889,  0.6227, -0.6471],\n",
      "        [-0.7174, -1.0516, -1.1512,  0.8065, -2.2778],\n",
      "        [-0.2217, -3.0190, -0.0596, -0.3587, -2.0306]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8256, 0.1280, 0.1325, 0.2055, 0.9196, 0.4239, 0.9843, 0.1485, 0.0122,\n",
      "        0.0034], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.0298,  1.6881,  1.7019, -0.0213, -0.8216],\n",
      "        [-1.5177,  0.1445,  0.5522,  0.5916,  0.1049],\n",
      "        [ 0.3941,  0.5039, -1.1807, -0.1878,  0.0256],\n",
      "        [-0.4733, -0.8958,  0.2275, -2.3963, -1.5770],\n",
      "        [ 0.5257,  0.2333, -0.2851, -1.1084, -0.7891],\n",
      "        [ 0.1811,  0.1535,  3.1537, -1.1862,  1.6745],\n",
      "        [-2.7996, -0.8029, -1.6441, -0.9651, -0.6577],\n",
      "        [-0.0219,  0.5652,  0.3985,  5.6227, -2.3494],\n",
      "        [-0.2562, -0.0633,  0.9799,  1.9074, -0.0739],\n",
      "        [-1.1841, -0.1999, -0.6323,  0.4361,  0.6251]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9294, 0.4689, 0.3906, 0.0060, 0.1941, 0.9816, 0.0010, 0.9854, 0.9237,\n",
      "        0.2779], grad_fn=<SigmoidBackward>)\n",
      "loss: 2.091020  [  150/  981]\n",
      "out: tensor([[-0.2217, -3.0189, -0.0596, -0.3586, -2.0305],\n",
      "        [-0.8536, -0.4851, -1.7121,  0.0107, -1.2770],\n",
      "        [-0.8742, -0.0355, -0.0927, -0.4870, -2.4397],\n",
      "        [-0.0736,  0.0223,  1.0798, -0.0182,  0.9981],\n",
      "        [ 0.1019,  0.0266, -0.5712,  0.0368,  1.4316],\n",
      "        [ 3.3496,  0.3402,  0.1595, -0.1876, -1.3802],\n",
      "        [ 1.7178,  1.7513, -0.1185, -0.2994, -0.7618],\n",
      "        [-0.4878, -0.1477, -0.5808, -0.0992,  1.7027],\n",
      "        [ 1.4392, -0.6778, -0.2267, -2.4826,  0.9361],\n",
      "        [-0.1346, -0.1390, -0.2406,  0.3831, -1.5300]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0034, 0.0132, 0.0193, 0.8817, 0.7361, 0.9073, 0.9080, 0.5956, 0.2666,\n",
      "        0.1596], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.2648, -0.9814, -0.2836,  2.1813, -0.6160],\n",
      "        [-1.5130,  0.0870,  0.9915,  0.9618, -0.1716],\n",
      "        [-0.2681,  1.2162,  0.2491,  0.3748,  2.7381],\n",
      "        [-0.0938, -0.1870, -0.3239,  2.1902,  0.9302],\n",
      "        [ 0.6415,  2.5165, -0.3094, -2.9262, -2.0793],\n",
      "        [ 0.0814, -0.0493, -0.3276, -0.0615, -1.6417],\n",
      "        [-0.0056,  1.8712, -0.7501,  0.0771,  0.0639],\n",
      "        [-0.4365,  0.8127, -0.5713, -0.8339,  0.3149],\n",
      "        [ 0.8804,  0.1937, -0.7761, -0.4152, -1.3114],\n",
      "        [-1.8062, -0.2448,  0.4053, -0.0236, -0.6812]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.5089, 0.5880, 0.9867, 0.9252, 0.1037, 0.1193, 0.7784, 0.3287, 0.1933,\n",
      "        0.0870], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.1771,  0.8498, -0.2173, -3.9602, -0.0153],\n",
      "        [ 0.2819, -0.0639,  0.0087,  0.5640, -1.8497],\n",
      "        [-0.4225, -0.0902,  0.0619,  0.0517, -1.2775],\n",
      "        [-0.1945, -0.1897, -1.4656,  0.9799, -0.0951],\n",
      "        [ 0.6425, -0.1172, -0.4166, -0.0399, -0.2624],\n",
      "        [ 0.5125, -0.7819,  1.9849,  0.2335,  0.0056],\n",
      "        [-0.1687, -0.2007, -0.7070, -0.1627,  0.0165],\n",
      "        [ 0.4255,  0.3226,  0.4633, -1.0270, -0.0634],\n",
      "        [-0.0377, -0.2285,  1.2258, -0.7563,  0.3639],\n",
      "        [-0.0231,  0.7443,  1.2302, -0.1042, -0.8183]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0405, 0.2575, 0.1575, 0.2759, 0.4517, 0.8759, 0.2275, 0.5302, 0.6381,\n",
      "        0.7367], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 2.0651e-01, -1.1093e-01,  4.0448e-01, -1.5830e-01,  1.1934e+00],\n",
      "        [ 2.4642e-01,  4.2005e-01, -4.9224e-02,  7.6764e-03, -1.2295e+00],\n",
      "        [ 5.9943e-01,  2.5038e-01,  3.7685e-01, -9.4412e-01,  8.4530e-01],\n",
      "        [ 8.5021e-01, -4.8300e-01,  4.8463e-01, -3.4227e-01, -2.1468e-01],\n",
      "        [ 4.3145e-01, -5.8589e-01, -3.8938e-01,  3.5280e-02, -6.8472e-02],\n",
      "        [ 3.3911e-02, -1.6209e+00,  7.1197e-01,  3.6498e-02,  4.8457e-02],\n",
      "        [-1.9103e-03, -3.0000e+00,  1.0584e-01, -1.2595e-01, -6.9160e-01],\n",
      "        [ 8.2650e-01, -2.1563e-01,  4.5142e-01,  4.7123e-01,  8.2422e-01],\n",
      "        [ 6.7355e-01,  9.4928e-03, -1.2568e-01, -3.9704e-02, -1.1043e+00],\n",
      "        [ 3.3568e-01, -2.2060e-01,  2.0788e+00, -3.9694e-01,  7.6033e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8228, 0.3533, 0.7554, 0.5732, 0.3596, 0.3122, 0.0238, 0.9135, 0.3574,\n",
      "        0.9281], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-1.2405,  0.4283, -0.3538, -0.5701, -0.7172],\n",
      "        [ 0.8096, -0.5689,  0.0258, -0.1401,  0.2716],\n",
      "        [-0.2732, -0.1485, -0.5503, -3.4115, -0.0072],\n",
      "        [ 0.0734, -0.0109, -1.2910,  0.2630,  0.1011],\n",
      "        [ 0.0296,  0.6662,  0.2985,  2.2680,  0.9281],\n",
      "        [ 1.1917, -1.6938,  0.9019,  1.1065,  0.4664],\n",
      "        [ 0.2391, -1.2292,  0.1965,  0.1974,  1.5724],\n",
      "        [ 0.3196,  0.2564,  0.2970,  0.8555,  0.0820],\n",
      "        [ 2.6637, -0.5747,  2.1125, -0.3121, -0.4559],\n",
      "        [ 0.7805, -0.7006, -0.9659,  0.1112,  0.1848]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0792, 0.5982, 0.0122, 0.2964, 0.9851, 0.8779, 0.7263, 0.8594, 0.9687,\n",
      "        0.3566], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.461212  [  200/  981]\n",
      "out: tensor([[-0.3579, -0.3383,  1.4139,  1.2875, -0.1359],\n",
      "        [-0.7420, -0.0845, -0.1291,  0.2996,  0.7116],\n",
      "        [ 0.6217, -0.1586,  0.2883,  0.1392, -0.0595],\n",
      "        [-0.5988,  2.2656, -0.0954,  0.0928, -0.1330],\n",
      "        [ 0.0234, -2.0120,  0.0872, -0.1106,  0.2495],\n",
      "        [-0.0317,  1.3541, -1.3504,  0.4229,  0.0826],\n",
      "        [ 0.4323,  0.1819, -2.9648,  0.1900,  0.3699],\n",
      "        [ 0.0659,  0.6194, -0.3630, -0.0866,  0.0570],\n",
      "        [-0.4118, -0.1605, -1.1069, -0.1341, -0.1100],\n",
      "        [-0.4020, -0.4435,  0.0917,  1.9801,  1.8781]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8664, 0.5139, 0.6966, 0.8222, 0.1465, 0.6172, 0.1430, 0.5727, 0.1275,\n",
      "        0.9571], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.5917,  0.2800,  1.7528, -0.0895,  0.4719],\n",
      "        [-0.5101, -0.4568, -3.2799,  0.6828, -1.8026],\n",
      "        [ 0.1345,  0.5380, -0.8830, -0.5005, -0.0633],\n",
      "        [ 0.9920,  3.0175, -0.0953,  0.1615, -1.5992],\n",
      "        [-0.3458, -0.0347,  0.4361, -0.0897, -0.2148],\n",
      "        [-0.8201, -0.3784, -0.4229, -1.5778,  0.3680],\n",
      "        [ 0.3154, -1.2012, -1.4246, -0.4681,  0.3011],\n",
      "        [ 0.0082, -0.2652, -0.4410,  0.1659,  0.2260],\n",
      "        [ 0.3874,  0.7639, -1.4831,  1.0292,  0.6495],\n",
      "        [ 0.3330, -0.4882, -0.5668, -0.1601, -0.4385]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9821, 0.0046, 0.3156, 0.9225, 0.4381, 0.0557, 0.0775, 0.4241, 0.7936,\n",
      "        0.2107], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.1832,  0.0612,  1.5084, -0.5684,  1.0273],\n",
      "        [-0.3495, -1.0681, -1.5880, -0.2444, -0.7333],\n",
      "        [ 0.3815, -0.0429,  1.2665, -0.3211, -0.1375],\n",
      "        [ 0.4659,  0.9909, -0.1198, -1.1382, -0.5499],\n",
      "        [ 0.1985,  0.0519, -0.0427, -0.1552, -1.6926],\n",
      "        [-0.1017, -0.4113,  0.3323, -0.2051,  0.8116],\n",
      "        [-0.2153,  0.4752, -0.0993,  0.6418,  0.4265],\n",
      "        [ 0.3839, -0.1414, -0.0343, -0.2319,  0.3190],\n",
      "        [ 0.4315, -0.5859, -0.3894,  0.0353, -0.0685],\n",
      "        [-0.5057,  0.0695,  0.7622,  1.5071, -0.5637]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9013, 0.0183, 0.7589, 0.4131, 0.1624, 0.6049, 0.7736, 0.5733, 0.3596,\n",
      "        0.7806], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.0093, -0.8399,  0.0476, -0.1314, -0.4931],\n",
      "        [-0.3992, -0.2724,  0.8927, -0.0952, -2.6280],\n",
      "        [-0.3752, -0.2458,  0.1460, -1.7869,  0.1154],\n",
      "        [ 0.8295, -0.0619, -0.8581,  0.9610, -2.2957],\n",
      "        [-0.7288,  0.0744,  0.6306,  0.0841, -0.1764],\n",
      "        [-0.0226, -1.5879,  0.1509, -0.2630,  0.2924],\n",
      "        [ 0.1820,  0.1175,  0.0152,  0.1970,  2.7983],\n",
      "        [-1.0332, -0.1427, -0.9264, -1.6528,  0.3285],\n",
      "        [ 1.1707,  0.2497, -1.9816,  0.7884,  0.3090],\n",
      "        [ 0.3789, -0.3038, -2.5192,  0.8498,  0.4585]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1937, 0.0757, 0.1047, 0.1938, 0.4710, 0.1931, 0.9648, 0.0315, 0.6309,\n",
      "        0.2431], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.0420, -0.0400,  0.4585,  0.3520,  1.3507],\n",
      "        [ 2.0449, -0.1588, -4.3165,  0.3421,  0.6692],\n",
      "        [-0.4771,  0.1166,  0.2934,  0.2891, -0.2743],\n",
      "        [-0.2756,  1.6733,  1.0654,  0.5334,  1.3560],\n",
      "        [ 0.3815, -0.0429,  1.2665, -0.3211, -0.1376],\n",
      "        [ 1.5597, -0.0465,  0.5427, -0.3812,  1.4674],\n",
      "        [-0.0056,  1.8711, -0.7501,  0.0771,  0.0639],\n",
      "        [ 0.1857, -0.1629, -0.3545, -0.2322,  0.4093],\n",
      "        [-0.0958, -0.8317, -0.3488, -0.6993,  0.8788],\n",
      "        [ 0.0684,  1.1846,  1.3983,  1.4690, -1.1595]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8889, 0.1948, 0.4870, 0.9873, 0.7588, 0.9586, 0.7784, 0.4614, 0.2503,\n",
      "        0.9508], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.400416  [  250/  981]\n",
      "out: tensor([[ 1.2861, -0.6294, -0.6913,  0.2313, -1.5994],\n",
      "        [ 0.5540,  0.0227,  1.7873, -0.9717,  1.5923],\n",
      "        [-0.4654, -0.6539, -0.9261,  0.1898,  0.5448],\n",
      "        [-1.4746, -1.6745,  0.1682,  0.0665, -4.9867],\n",
      "        [-0.6174,  0.8727,  0.8212, -0.7421, -0.9969],\n",
      "        [ 0.1364, -0.0944,  0.4411,  0.3770, -0.8029],\n",
      "        [ 1.5156,  0.9749, -0.7614,  0.2038,  2.9897],\n",
      "        [ 1.7954, -0.1640,  2.7645, -0.1073, -0.1218],\n",
      "        [-0.0938, -1.3243,  0.2146, -0.1225,  0.3232],\n",
      "        [ 0.3858,  0.1675, -0.0087,  0.5190,  0.0141]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([1.9741e-01, 9.5187e-01, 2.1235e-01, 3.7023e-04, 3.4016e-01, 5.1427e-01,\n",
      "        9.9277e-01, 9.8474e-01, 2.6840e-01, 7.4605e-01],\n",
      "       grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-1.2934,  3.4584,  0.0139,  0.0632,  3.4676],\n",
      "        [-0.2672, -1.7581, -0.1432, -0.5389,  0.1351],\n",
      "        [ 0.7376,  0.3593,  0.8749,  0.2045, -0.9569],\n",
      "        [ 0.7561,  0.1348,  0.9647, -0.3229, -0.1975],\n",
      "        [ 0.0684,  1.1846,  1.3983,  1.4690, -1.1595],\n",
      "        [ 2.8875,  1.0997,  0.1485, -0.4261,  0.4942],\n",
      "        [ 1.1372,  0.3502, -0.1106,  0.4199, -2.1569],\n",
      "        [-0.4439, -1.1220,  0.2553, -1.1409, -1.2726],\n",
      "        [-0.1333, -0.3306,  0.1734, -0.2478, -1.0377],\n",
      "        [ 0.3091, -0.2197,  2.9861, -0.1186,  0.0863]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9967, 0.0709, 0.7720, 0.7917, 0.9508, 0.9853, 0.4109, 0.0236, 0.1714,\n",
      "        0.9545], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.3799,  0.0115,  0.3243,  1.2869,  1.0778],\n",
      "        [ 0.6903,  0.3914, -0.0433,  0.5824, -2.1370],\n",
      "        [ 0.8079, -0.2153,  0.4571, -0.1222,  0.0630],\n",
      "        [ 0.4482, -0.3851,  1.0476, -0.0918,  0.0931],\n",
      "        [ 0.3245,  1.0264, -0.3718,  0.5544,  0.8000],\n",
      "        [ 0.0536, -0.8638,  0.3533, -0.1204, -0.1668],\n",
      "        [-0.7052,  0.8162,  1.1409,  0.2081,  0.3348],\n",
      "        [-0.0317,  1.3540, -1.3504,  0.4229,  0.0826],\n",
      "        [ 0.3624, -0.1779,  2.7167, -0.0202,  0.0633],\n",
      "        [-0.7177,  0.8323,  0.7633, -0.0584, -2.1203]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9106, 0.3738, 0.7292, 0.7525, 0.9116, 0.3221, 0.8575, 0.6172, 0.9500,\n",
      "        0.2141], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.0174,  0.2475, -0.3017, -0.3490, -1.0768],\n",
      "        [-0.0409,  0.0921, -0.0386, -0.8050,  1.3277],\n",
      "        [-0.1996, -0.7816,  0.9831, -1.1276,  0.8999],\n",
      "        [ 0.0194,  0.3633,  0.0124,  0.7255,  0.6750],\n",
      "        [-0.1174, -0.1131,  0.1063,  0.0127,  0.3442],\n",
      "        [-0.4161, -0.1330,  0.2678, -1.6252, -0.2388],\n",
      "        [-0.0875, -0.4697,  0.0643,  0.3236,  0.1457],\n",
      "        [ 0.1339,  1.0730, -0.0474,  1.0352, -0.5231],\n",
      "        [-1.9019,  2.1168,  0.4845, -0.0548,  0.3078],\n",
      "        [-0.0956, -0.2633,  0.9762,  0.1894,  0.1352]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1881, 0.6307, 0.4438, 0.8576, 0.5579, 0.1048, 0.4941, 0.8418, 0.7216,\n",
      "        0.7195], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-5.0647e-01,  3.3530e-01,  6.0288e-01,  1.6919e-01, -2.1011e+00],\n",
      "        [ 1.1118e-01,  4.6073e+00,  3.0633e+00,  8.2635e-04, -7.3429e-01],\n",
      "        [ 1.0513e+00,  1.5839e+00,  1.4567e-01, -7.4029e-01,  2.7545e-01],\n",
      "        [-3.4010e-01, -3.5898e-01, -2.3816e+00,  9.3592e-01,  4.4908e-01],\n",
      "        [-1.2512e+00, -8.9271e-02, -5.4316e-01, -4.6394e-01,  4.5635e-01],\n",
      "        [-1.1767e+00, -2.8461e-01, -3.7340e-01, -6.8727e-03,  7.5007e-01],\n",
      "        [ 7.1503e-01,  2.8990e+00, -4.7352e-01, -3.0588e-01, -1.3628e+00],\n",
      "        [-3.8549e-01, -1.4107e-01,  3.1970e-01,  1.2478e-01,  7.9872e-01],\n",
      "        [-3.3944e-01, -4.4977e-01, -3.1499e-01,  1.1272e+00, -2.1064e-01],\n",
      "        [ 2.0618e+00, -9.3854e-01, -2.4047e+00,  1.4765e+00,  8.9902e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1824, 0.9991, 0.9102, 0.1550, 0.1311, 0.2513, 0.8133, 0.6719, 0.4532,\n",
      "        0.5708], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.566817  [  300/  981]\n",
      "out: tensor([[-1.2810e-01,  1.7896e+00,  6.0429e-01,  9.8440e-01, -2.0311e+00],\n",
      "        [ 8.2388e-02, -9.2813e-01,  2.1715e-03,  9.2343e-01, -4.7630e-01],\n",
      "        [-6.8884e-02,  1.9537e+00,  9.1503e-01, -1.2656e+00,  1.3789e+00],\n",
      "        [ 5.3399e-01, -2.5828e-01, -1.6817e+00,  5.3600e-02,  1.5014e+00],\n",
      "        [ 5.2994e-01, -8.5997e-02,  1.2885e+00, -6.9228e-01,  2.9538e+00],\n",
      "        [ 1.1158e+00,  4.9269e-01, -5.9177e-01,  1.9433e-02, -6.8834e-01],\n",
      "        [-5.2177e-01,  6.7490e-02,  3.6898e-01,  5.0683e-02, -1.1041e+00],\n",
      "        [-5.3086e-02, -6.0397e-01,  1.4826e+00, -6.3364e-02,  1.1031e-01],\n",
      "        [-5.4241e-01,  8.2278e-01,  2.6112e+00, -6.6909e-02, -5.1924e-01],\n",
      "        [-2.5459e-01, -6.1191e-01, -8.7632e-02,  6.4387e-01, -6.0309e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7719, 0.4022, 0.9485, 0.5372, 0.9819, 0.5861, 0.2426, 0.7053, 0.9093,\n",
      "        0.2863], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.6057,  0.3727,  0.2928,  1.2850,  0.6174],\n",
      "        [-0.3296, -0.7808,  0.0553,  0.1081, -0.1184],\n",
      "        [ 0.5662,  0.1572,  0.0412,  0.0101, -0.3729],\n",
      "        [-0.2546, -0.6119, -0.0876,  0.6439, -0.6031],\n",
      "        [ 0.4160,  0.4809,  0.4378,  0.3059, -0.0769],\n",
      "        [-1.6309, -0.0706, -3.9458, -0.5701,  3.0866],\n",
      "        [-0.6525, -0.2379, -0.0309,  0.3599,  1.5017],\n",
      "        [-0.0691,  2.8267,  0.3204,  0.5451, -0.2443],\n",
      "        [-0.0154, -0.0225,  0.2762,  1.2362, -1.2186],\n",
      "        [-0.0127, -0.3713,  0.1414, -3.1504,  0.6419]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8768, 0.2563, 0.5991, 0.2863, 0.8269, 0.0419, 0.7192, 0.9670, 0.5636,\n",
      "        0.0600], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.0976, -0.2208, -0.3559,  1.1175,  1.6757],\n",
      "        [ 0.8047, -0.0432, -0.2146, -0.9839,  0.0749],\n",
      "        [ 0.0688, -0.5403, -1.1716, -1.2382,  0.3222],\n",
      "        [-0.6297,  0.3464,  0.0573, -0.9885,  1.0019],\n",
      "        [-0.9332,  0.9737, -0.0422, -0.3624,  0.8204],\n",
      "        [ 0.2468, -1.6397,  0.5135, -1.5576, -0.3344],\n",
      "        [ 0.6666, -0.1241,  0.7394,  0.2239,  0.0562],\n",
      "        [ 1.2368,  0.1623, -0.2393, -0.8273,  0.4303],\n",
      "        [-0.4963,  0.3036, -1.6712,  0.6299,  0.9485],\n",
      "        [-0.2985,  1.3067,  0.5402, -1.1323,  0.5106]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9100, 0.4105, 0.0718, 0.4471, 0.6121, 0.0589, 0.8266, 0.6819, 0.4291,\n",
      "        0.7164], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 2.7084, -0.4214,  0.0541, -0.4287, -0.0770],\n",
      "        [ 0.4286,  0.1329, -0.3283, -1.3768,  0.6230],\n",
      "        [ 0.4322,  0.1819, -2.9647,  0.1900,  0.3699],\n",
      "        [-0.4160,  0.2031,  0.0142, -1.0014,  0.0305],\n",
      "        [ 0.7016,  0.8635, -0.1340, -0.5367,  0.8920],\n",
      "        [ 0.1570, -1.3719, -0.4037,  0.4144,  0.0474],\n",
      "        [ 0.2260,  0.4121, -0.3900,  0.0227,  0.3461],\n",
      "        [-0.0409,  0.0921, -0.0386, -0.8051,  1.3277],\n",
      "        [ 1.1307, -0.7356,  0.3036,  0.7394,  0.3679],\n",
      "        [-0.1621, -0.3659,  0.2372,  0.7811,  1.3999]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8624, 0.3727, 0.1430, 0.2369, 0.8565, 0.2393, 0.6495, 0.6307, 0.8589,\n",
      "        0.8688], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.8786,  0.9110,  0.3097,  0.9744, -0.4521],\n",
      "        [ 0.7994,  0.1848,  4.5382, -0.0654,  0.8677],\n",
      "        [-0.2072, -0.6072,  0.3896, -0.2138,  1.1040],\n",
      "        [ 1.1045,  0.4911, -1.7240,  0.2683, -2.2613],\n",
      "        [ 0.0496,  0.1089, -0.3529, -0.3096, -0.2728],\n",
      "        [-0.1255, -2.1379, -0.2402,  1.1309, -0.2112],\n",
      "        [ 0.8079, -0.2153,  0.4571, -0.1222,  0.0630],\n",
      "        [ 0.0831,  0.1263,  0.1543, -0.1275, -0.2214],\n",
      "        [ 0.1424, -0.4652,  0.3993, -0.6598, -0.9782],\n",
      "        [ 1.4727,  0.8864, -1.2769, -1.2058,  0.4475]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9322, 0.9982, 0.6143, 0.1070, 0.3150, 0.1702, 0.7292, 0.5037, 0.1734,\n",
      "        0.5803], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.299608  [  350/  981]\n",
      "out: tensor([[-0.5524, -1.6476,  0.1041, -0.4873, -0.7330],\n",
      "        [-0.2716, -0.3552,  0.6233, -0.1429, -0.8529],\n",
      "        [-0.6238, -2.0291,  0.8413, -0.5715, -0.3193],\n",
      "        [ 0.7650, -0.0205, -0.1313, -0.0925, -0.5675],\n",
      "        [ 0.5340, -0.2583, -1.6817,  0.0536,  1.5014],\n",
      "        [ 0.4752, -3.4449, -0.0067,  0.6685, -0.3686],\n",
      "        [ 0.0795,  0.4247,  1.0675,  0.6643,  0.3976],\n",
      "        [ 0.2886, -1.8639,  0.2039,  0.4735, -0.0463],\n",
      "        [-0.2115,  0.6450,  1.4051,  0.6083, -0.1649],\n",
      "        [ 0.5209, -3.1244, -0.2858,  0.3705, -0.1020]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0350, 0.2691, 0.0628, 0.4883, 0.5372, 0.0644, 0.9330, 0.2801, 0.9074,\n",
      "        0.0678], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-3.4246e-01,  4.8629e-01,  1.3918e+00,  3.6740e+00,  5.7587e-01],\n",
      "        [-1.0043e-01,  1.7547e+00, -2.0745e-02,  2.0955e+00,  1.9737e-02],\n",
      "        [ 5.1305e-01, -7.6774e-01, -9.7690e-01,  2.5204e-01, -5.9463e-02],\n",
      "        [-3.0774e-01, -1.5604e+00, -3.3612e+00,  3.8964e+00, -3.0603e-01],\n",
      "        [ 4.2832e-01, -4.6588e-01, -1.3991e+00,  2.6514e-02, -8.4249e-02],\n",
      "        [ 2.0579e-01, -1.1798e+00, -1.1521e-01,  4.3385e-01, -2.6039e-01],\n",
      "        [-1.0105e+00, -3.1900e-02, -2.0651e+00, -9.7250e-01,  1.1905e-01],\n",
      "        [-1.9084e-03, -2.9999e+00,  1.0583e-01, -1.2596e-01, -6.9159e-01],\n",
      "        [-4.2224e-01,  5.1165e-01,  1.6943e-01, -1.1653e+00, -2.8107e-01],\n",
      "        [-2.4276e-01,  5.1436e-01,  1.2736e+00, -1.7178e-01, -8.3171e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9969, 0.9770, 0.2613, 0.1626, 0.1833, 0.2858, 0.0187, 0.0238, 0.2337,\n",
      "        0.7842], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 3.8849e-03,  2.1178e+00, -3.9170e-01,  1.3252e-01,  7.6011e-01],\n",
      "        [-1.2679e-01,  2.1596e+00, -2.4978e+00,  8.1254e-02, -9.3598e-01],\n",
      "        [ 4.6869e-01,  5.4479e-01, -2.0681e-01, -1.5669e-01,  1.2773e+00],\n",
      "        [-5.7847e-01, -3.9026e-01, -4.3261e-01, -1.0927e+00,  5.5498e-01],\n",
      "        [-2.5717e-02, -2.0630e+00,  2.8494e-02, -1.1196e+00, -3.1442e-01],\n",
      "        [ 7.3493e-01,  2.4279e-01, -3.5680e-02,  4.7049e-01,  6.5048e-01],\n",
      "        [-1.2606e-02,  9.0113e-01,  2.7431e-01,  1.2254e-03,  5.9876e-01],\n",
      "        [ 1.6068e-02,  1.1357e+00,  5.8484e-01, -1.3928e-01, -1.2391e+00],\n",
      "        [ 5.7534e-01,  3.9451e+00,  1.0910e+00, -9.2895e-02, -2.3123e+00],\n",
      "        [ 1.1759e+00,  3.9221e-02, -2.0753e+00,  2.4246e+00, -4.0260e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9323, 0.2109, 0.8729, 0.1258, 0.0295, 0.8873, 0.8536, 0.5886, 0.9611,\n",
      "        0.7617], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-5.9413e-02, -2.5163e+00,  5.5529e-01, -3.0926e+00,  3.5152e-01],\n",
      "        [ 1.5318e-02, -9.3803e-01,  5.1969e-01, -1.0072e-01, -9.2184e-01],\n",
      "        [ 8.0278e-01,  2.7218e-01,  8.5434e-01,  3.1776e-01,  2.4834e+00],\n",
      "        [-1.7489e+00, -2.0176e-01, -1.9162e+00,  2.9340e-01, -2.2183e-01],\n",
      "        [ 2.6108e+00,  9.5620e-02,  6.8183e-01,  1.1864e+00,  5.4834e-01],\n",
      "        [-2.9850e-01,  1.3067e+00,  5.4022e-01, -1.1323e+00,  5.1056e-01],\n",
      "        [-5.5503e-01,  5.0401e-01,  2.8892e-01,  1.2219e+00, -6.9718e-01],\n",
      "        [ 2.5418e-01, -2.7871e-03, -6.7417e-01, -1.8696e-01,  2.0776e+00],\n",
      "        [ 2.0190e-01,  2.9465e-01,  9.6579e-01, -1.6646e+00, -1.7154e-01],\n",
      "        [-2.7496e-02, -7.2713e-01,  3.7238e-01, -1.9582e-01, -4.8013e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0085, 0.1938, 0.9913, 0.0220, 0.9941, 0.7164, 0.6819, 0.8127, 0.4076,\n",
      "        0.2577], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-2.2822e-02, -4.4290e-01,  8.5273e-02,  1.2033e-01, -5.6435e-01],\n",
      "        [-1.0656e-03,  2.6187e+00, -2.5529e-02, -6.6043e-01,  2.3006e-01],\n",
      "        [-7.1745e-01, -1.0516e+00, -1.1512e+00,  8.0652e-01, -2.2778e+00],\n",
      "        [ 3.8984e+00,  7.3153e-01,  2.2182e-01, -2.3551e-01, -2.0139e+00],\n",
      "        [-7.7279e-01, -1.1096e+00,  2.0677e-02,  4.0082e-01,  1.9447e+00],\n",
      "        [ 1.9132e-01,  1.4367e+00,  2.2197e-01, -3.5053e-01, -1.1308e-01],\n",
      "        [ 8.2006e-01,  2.9730e-01,  1.6495e-01, -2.2620e-01, -1.6016e+00],\n",
      "        [-4.0205e-01, -7.5761e-01,  8.3172e-01,  8.7528e-01,  5.5715e-01],\n",
      "        [-1.9788e+00, -5.7852e-01,  1.0667e+00,  1.1938e-02, -7.7064e-01],\n",
      "        [ 2.2166e-01, -5.5872e-02,  2.1593e+00, -4.6816e-02, -1.0415e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3048, 0.8968, 0.0122, 0.9310, 0.6186, 0.8000, 0.3669, 0.7511, 0.0954,\n",
      "        0.7750], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.104517  [  400/  981]\n",
      "out: tensor([[-1.3836e+00,  5.6214e-01, -3.0762e-01,  5.5250e-02, -5.3386e-01],\n",
      "        [ 3.4255e-02, -1.4152e+00, -8.8838e-01,  4.0165e-01, -1.6409e-01],\n",
      "        [-2.2310e-02,  2.0849e-01, -1.5752e-01, -2.1842e+00, -6.3187e-02],\n",
      "        [-3.6803e-01,  2.4669e+00,  8.2245e-01, -3.3336e-01, -1.4592e+00],\n",
      "        [ 1.6604e-01,  8.8542e-01,  6.1071e-01, -1.4473e+00, -9.2071e-01],\n",
      "        [-7.7388e-01, -4.1046e-01,  7.8469e-01,  5.0939e-01,  2.2019e+00],\n",
      "        [-2.9143e-01, -6.2498e-02,  8.1466e-01, -3.8193e-01,  6.9144e-01],\n",
      "        [ 2.1083e-03, -2.3950e+00, -1.5280e-01, -1.3916e+00,  2.7093e-01],\n",
      "        [-9.9637e-01, -1.2071e-01,  7.7897e-01,  1.7102e-01,  7.3027e-01],\n",
      "        [-5.2488e-01, -4.6861e-01, -4.3121e-01,  1.1372e+00,  2.9115e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1669, 0.1159, 0.0981, 0.7556, 0.3305, 0.9098, 0.6836, 0.0249, 0.6372,\n",
      "        0.4357], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.2225,  1.8963, -0.5890, -0.1855,  1.7168],\n",
      "        [ 0.5251,  0.5106,  0.4647, -0.9841,  2.0391],\n",
      "        [ 1.9776,  0.3057, -0.0907,  0.0295,  0.7649],\n",
      "        [ 0.3085, -0.0963, -0.0841, -0.5429,  0.5538],\n",
      "        [ 1.0250,  0.0483, -0.3465, -0.3229, -0.1389],\n",
      "        [ 0.3571,  2.0173,  0.4865,  0.3420, -0.7192],\n",
      "        [ 1.2907, -1.0625,  0.7994, -0.0788,  1.0411],\n",
      "        [ 0.5131, -0.7677, -0.9769,  0.2520, -0.0595],\n",
      "        [-0.3607, -1.7357, -0.1681, -0.1921, -0.2999],\n",
      "        [-0.3909,  0.0262, -0.0378, -0.2945, -0.6413]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9553, 0.9279, 0.9520, 0.5347, 0.5659, 0.9230, 0.8797, 0.2614, 0.0597,\n",
      "        0.2078], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 6.8032e-01,  2.2037e-01,  1.5100e+00, -6.5023e-02,  1.0734e-01],\n",
      "        [ 3.3992e-01, -1.2931e-01, -2.4315e+00, -6.7266e-02, -5.5435e-01],\n",
      "        [ 1.8618e-01, -1.5179e-01,  2.4918e+00,  5.7167e-01,  5.0724e-01],\n",
      "        [-1.1583e-01,  7.9485e-01, -6.3433e-02,  4.0653e-04, -3.2042e-01],\n",
      "        [ 1.8778e+00, -2.1488e-01, -3.4417e-01,  1.4762e-01,  1.4658e-01],\n",
      "        [-6.2200e-01, -1.1494e+00, -9.5058e-01,  4.5281e-01,  2.5951e+00],\n",
      "        [ 3.0106e-02, -1.7209e-01,  1.7742e-01, -4.3454e-01,  1.5016e+00],\n",
      "        [-2.1245e+00,  4.9379e-01, -1.3475e+00,  8.7646e-03,  2.2191e-01],\n",
      "        [ 1.2022e-01,  4.2380e-01, -9.8177e-01,  3.5666e-01,  8.0467e-03],\n",
      "        [ 4.5025e-01, -3.8772e-01,  4.8509e-01, -5.6262e-02,  7.8256e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9208, 0.0551, 0.9735, 0.5734, 0.8338, 0.5808, 0.7507, 0.0602, 0.4817,\n",
      "        0.7814], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.1373, -0.9225,  0.0226, -0.0512,  0.1351],\n",
      "        [-0.4054,  0.1896,  0.4040, -0.3986, -1.4337],\n",
      "        [ 0.1119,  0.0196,  0.1952, -0.0099, -1.7232],\n",
      "        [-0.0232,  0.7443,  1.2302, -0.1042, -0.8183],\n",
      "        [-0.8429, -0.0430,  0.0703, -0.0847, -1.1944],\n",
      "        [ 0.3929, -0.3751,  0.1801, -0.3291, -1.0488],\n",
      "        [ 0.4482, -0.3851,  1.0475, -0.0918,  0.0931],\n",
      "        [-0.0147,  0.0028, -1.8820, -1.4442,  0.0859],\n",
      "        [-0.0223, -0.1369, -0.4910, -1.1429, -1.1188],\n",
      "        [ 0.1748,  0.3921, -1.0172, -0.0206,  0.4180]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3365, 0.1619, 0.1968, 0.7367, 0.1096, 0.2350, 0.7525, 0.0372, 0.0516,\n",
      "        0.4868], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.0638, -0.5955,  0.0104,  0.6226,  1.3764],\n",
      "        [ 0.0544, -1.3441,  0.3694,  0.5557, -0.2564],\n",
      "        [-0.4657, -0.5044,  0.3229, -0.2645,  1.0058],\n",
      "        [-0.4931,  0.0063,  0.2579,  0.1790,  1.7831],\n",
      "        [-0.1609,  0.7977, -0.0928,  1.2872,  1.3522],\n",
      "        [ 0.2517, -1.7326,  1.4763, -0.4467,  0.1174],\n",
      "        [ 0.3252,  0.4434,  0.2541, -2.8351,  0.0337],\n",
      "        [-0.2836,  0.5547, -1.9370, -0.0814, -0.9379],\n",
      "        [-0.0856, -0.5757, -0.1493, -0.2412, -1.8481],\n",
      "        [ 0.3091,  0.3674, -0.7907, -3.3492,  0.3319]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8142, 0.3496, 0.5235, 0.8498, 0.9602, 0.4173, 0.1445, 0.0638, 0.0522,\n",
      "        0.0418], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.403836  [  450/  981]\n",
      "out: tensor([[ 1.5328,  0.8852, -0.5804,  0.1408,  1.7021],\n",
      "        [ 0.6508,  1.4570,  2.2437, -0.2460,  1.1807],\n",
      "        [ 0.0553,  0.7524,  0.5217, -0.9653, -1.9504],\n",
      "        [ 2.1393,  0.5624, -0.3418,  0.2475, -1.1427],\n",
      "        [ 0.1342, -0.0671, -0.0906,  0.3065, -2.6314],\n",
      "        [-1.9836,  0.5128, -0.4117, -1.3360, -0.2680],\n",
      "        [-0.1333, -0.1677,  0.9945,  0.0316, -0.7050],\n",
      "        [-0.0095,  2.0951,  0.3481, -0.0367,  0.0780],\n",
      "        [-0.9207, -0.0709, -2.1198,  0.5455, -0.2756],\n",
      "        [-0.0794, -3.3661, -1.0290, -0.4096, -0.1709]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9754, 0.9950, 0.1699, 0.8123, 0.0872, 0.0297, 0.5050, 0.9224, 0.0551,\n",
      "        0.0063], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.3288, -0.2817,  1.3468, -0.2818, -0.0129],\n",
      "        [ 2.4722, -0.1171, -0.2093, -0.3215, -0.6554],\n",
      "        [ 0.0664, -0.9983,  0.2212,  0.0059, -1.5626],\n",
      "        [-0.3704,  0.1329, -0.5981,  0.7240,  0.1254],\n",
      "        [ 0.6160, -1.9702, -0.1401, -1.2040, -0.6176],\n",
      "        [-2.7892,  1.2898,  0.2286,  0.0908, -0.5792],\n",
      "        [-0.0232,  0.7443,  1.2302, -0.1042, -0.8183],\n",
      "        [ 0.0840, -0.4603,  0.6933, -0.0575, -0.6281],\n",
      "        [-0.1885,  0.1779, -0.3050, -0.0252, -2.1205],\n",
      "        [-0.0088, -0.0303, -0.2421, -0.0868,  1.0304]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7501, 0.7630, 0.0939, 0.5034, 0.0350, 0.1469, 0.7367, 0.4089, 0.0786,\n",
      "        0.6598], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.4034,  0.0446,  0.5109, -2.1692, -0.2133],\n",
      "        [-1.6627,  0.3416,  1.4533, -0.2992,  0.1187],\n",
      "        [ 0.2895, -0.7640, -0.3608, -0.1102, -1.7079],\n",
      "        [-0.0833,  3.0131, -0.0086,  1.3944, -0.2965],\n",
      "        [-0.6823, -0.5974, -0.1164, -0.1083, -2.8818],\n",
      "        [-0.0594, -2.5162,  0.5553, -3.0925,  0.3515],\n",
      "        [ 0.3418, -0.6088,  2.0983,  0.0056, -0.1672],\n",
      "        [-1.3789,  0.0108, -0.6362,  0.4008, -0.2698],\n",
      "        [-0.0275, -0.7272,  0.3724, -0.1958, -0.4801],\n",
      "        [ 1.0902, -0.0802,  1.5005,  0.9402, -0.4365]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1941, 0.4879, 0.0658, 0.9823, 0.0123, 0.0085, 0.8415, 0.1332, 0.2576,\n",
      "        0.9532], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 9.4098e-01,  3.5012e-01,  8.7835e-01, -3.5233e-02,  2.4636e+00],\n",
      "        [-3.0357e-01,  2.7087e-01, -9.7686e-01, -8.7216e-03, -1.7686e-02],\n",
      "        [-3.1625e-02,  7.4152e-01, -3.8724e-01, -4.3540e-01,  1.5589e+00],\n",
      "        [ 6.5635e-01,  2.2919e-01, -1.6289e+00, -1.2040e-01, -2.3337e-01],\n",
      "        [ 1.6518e+00,  2.0180e-01, -3.9012e+00, -1.5235e-01, -7.3383e-01],\n",
      "        [-8.4114e-01,  1.0586e+00, -3.5605e-01, -1.7658e+00,  2.4160e+00],\n",
      "        [ 2.1107e-03, -2.3949e+00, -1.5280e-01, -1.3915e+00,  2.7095e-01],\n",
      "        [ 6.1062e-01,  1.4469e-02,  3.7607e-01,  5.2955e-01,  3.8396e-02],\n",
      "        [-6.0209e-01,  1.9642e+00, -4.2719e-02, -7.3335e-01,  1.3533e+00],\n",
      "        [-1.3970e+00,  7.6914e-01,  3.1721e-01, -1.4848e-01,  6.8248e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9900, 0.2619, 0.8094, 0.2503, 0.0505, 0.6252, 0.0249, 0.8277, 0.8743,\n",
      "        0.5556], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.4101,  0.6792, -3.3642,  0.6971,  1.3391],\n",
      "        [-0.3529,  0.0352,  0.0537,  0.0754, -0.2465],\n",
      "        [-0.2428,  0.5143,  1.2736, -0.1718, -0.0832],\n",
      "        [ 1.2697, -0.0874, -4.8976,  0.1603, -1.1055],\n",
      "        [-0.2370, -0.1908, -0.4191,  0.9409,  0.0592],\n",
      "        [ 0.0582,  1.8466, -0.0128, -0.0234,  0.4067],\n",
      "        [-1.2726,  0.5234,  0.3528,  0.4146, -1.5277],\n",
      "        [ 0.5012,  0.6313, -0.2001,  0.1106,  1.7796],\n",
      "        [-1.4849,  0.8581, -0.1625,  0.3309, -1.7303],\n",
      "        [-0.9245, -2.3796, -0.3943,  0.7340,  1.4373]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.2575, 0.3929, 0.7842, 0.0094, 0.5382, 0.9068, 0.1810, 0.9439, 0.1008,\n",
      "        0.1784], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.020166  [  500/  981]\n",
      "out: tensor([[-7.0878e-01,  1.7085e+00, -1.5611e-01,  7.1052e-01,  1.0168e+00],\n",
      "        [ 1.5206e-02, -2.3820e-03, -1.0554e+00, -1.1638e+00, -1.2980e-02],\n",
      "        [ 1.3469e+00, -1.1890e+00,  3.1674e-01, -1.0093e-01,  8.9687e-01],\n",
      "        [ 1.1184e+00, -1.6932e-02, -6.8521e-01,  2.8882e-01, -4.0802e-01],\n",
      "        [ 1.0137e-01,  8.5703e-01, -2.1024e-02,  1.4692e+00, -5.8072e-02],\n",
      "        [ 1.1155e+00, -1.1077e+00,  6.6192e-01, -7.5422e-01, -9.9256e-01],\n",
      "        [ 4.4573e-01,  1.0283e+00,  9.5437e-01, -1.1814e-01, -1.4242e+00],\n",
      "        [ 3.0128e-02, -2.0421e+00, -1.1502e+00,  2.3501e-04, -5.6638e-01],\n",
      "        [-4.4340e-02,  1.3473e-03, -4.2208e-01, -7.1816e-01,  2.0062e-01],\n",
      "        [ 9.4182e-01, -9.8469e-01,  2.1748e+00, -1.9957e-02,  6.7387e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9290, 0.0980, 0.7808, 0.5737, 0.9128, 0.2541, 0.7081, 0.0235, 0.2724,\n",
      "        0.8984], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.1233,  0.8429, -0.3675, -0.1253,  0.1640],\n",
      "        [-0.9226,  0.6974, -0.3454,  1.6088,  0.4201],\n",
      "        [ 0.9162,  0.6112,  0.2124,  0.0358,  0.1851],\n",
      "        [ 0.4900,  1.2672, -0.4068,  0.0822, -0.0352],\n",
      "        [ 0.4606, -0.0283,  0.1474, -1.1005, -2.0090],\n",
      "        [-2.1574,  0.9600,  0.4207,  0.0540, -0.0959],\n",
      "        [-0.0773, -0.1925, -0.1448,  0.1099, -0.4669],\n",
      "        [-0.2556,  1.8065,  0.0800, -0.2424,  0.1828],\n",
      "        [-0.0420, -0.0400,  0.4586,  0.3520,  1.3506],\n",
      "        [-0.5863, -0.5463,  0.1876, -0.1058,  1.1014]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.5965, 0.8113, 0.8766, 0.8018, 0.0738, 0.3061, 0.3161, 0.8280, 0.8889,\n",
      "        0.5127], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.3196,  0.2564,  0.2970,  0.8556,  0.0819],\n",
      "        [ 0.1087,  0.0210,  0.2236, -0.5470, -1.6299],\n",
      "        [ 0.5580,  0.1052,  1.5607,  0.7905,  0.0938],\n",
      "        [ 0.9296, -0.1694,  0.3922,  1.8391,  0.5374],\n",
      "        [ 0.0225,  1.0244, -0.0251, -0.3503, -0.0828],\n",
      "        [-0.4773,  2.0647,  0.5294, -0.2510,  0.5011],\n",
      "        [-0.6684,  0.1170,  0.2994,  0.3414,  0.4314],\n",
      "        [ 0.8536,  1.1748, -0.1379, -0.0213, -0.1131],\n",
      "        [-1.1278,  0.4924,  0.0733, -0.1664, -1.4766],\n",
      "        [ 0.3941,  0.5038, -1.1806, -0.1878,  0.0256]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8594, 0.1390, 0.9572, 0.9715, 0.6431, 0.9143, 0.6273, 0.8527, 0.0993,\n",
      "        0.3906], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.2729,  3.2623, -0.2162,  0.7557,  1.6165],\n",
      "        [ 0.0315, -0.1798,  1.2702,  0.1691, -0.7864],\n",
      "        [-0.4666,  0.1034, -2.9646, -0.3854, -1.1471],\n",
      "        [-0.7429,  2.6933, -0.1310,  0.1322, -0.8858],\n",
      "        [ 1.3677, -0.1256,  0.7182, -0.7033,  0.2963],\n",
      "        [ 1.7914,  0.3186,  2.1778,  0.0181, -0.2793],\n",
      "        [-0.3712, -0.9558,  0.1130, -0.7335, -2.1164],\n",
      "        [-0.2601, -0.0294, -1.0522, -0.0922, -0.2563],\n",
      "        [ 0.0359, -1.3035, -2.3657, -0.2379, -0.8540],\n",
      "        [-0.0205,  3.1982, -0.0507, -0.0250, -2.0329]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9988, 0.6235, 0.0077, 0.7438, 0.8254, 0.9825, 0.0169, 0.1557, 0.0088,\n",
      "        0.7444], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.0661, -0.0941, -0.2052,  0.3675, -1.1247],\n",
      "        [ 0.0118, -1.8951, -0.0681,  0.1649, -0.3342],\n",
      "        [-1.6690,  1.3722, -0.2488,  0.4560, -1.8782],\n",
      "        [-0.5480,  0.6818, -0.6317, -0.8106, -1.1267],\n",
      "        [-0.3360,  0.0078,  0.1891, -0.0427, -1.1082],\n",
      "        [ 1.9212,  0.0143, -0.4908, -0.8735,  0.0083],\n",
      "        [ 0.9198,  1.5697, -0.1277,  1.1775,  0.6748],\n",
      "        [-0.0679, -2.9611, -2.5908,  1.3472, -1.2353],\n",
      "        [ 0.2606, -0.0310,  0.4343, -0.1037,  1.9472],\n",
      "        [-0.9014,  1.3404,  3.0194,  0.4447,  0.0730]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.2455, 0.1071, 0.1226, 0.0805, 0.2159, 0.6410, 0.9854, 0.0040, 0.9247,\n",
      "        0.9816], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.695901  [  550/  981]\n",
      "out: tensor([[-0.9332,  0.9737, -0.0422, -0.3623,  0.8203],\n",
      "        [ 0.0915,  0.2962, -0.7722,  0.9328,  0.1159],\n",
      "        [ 0.0241, -0.1303,  0.1568, -0.5425,  1.6987],\n",
      "        [ 0.2646,  2.7342, -1.6101,  0.0311, -0.1385],\n",
      "        [ 0.2767, -1.0196, -1.2021, -0.2532, -0.7543],\n",
      "        [ 1.7141,  0.3028,  0.1137,  0.4594,  0.9148],\n",
      "        [-0.7340, -0.0736, -0.6249, -0.6806, -0.0354],\n",
      "        [ 1.3849,  0.2764,  0.0138,  0.1065, -1.2062],\n",
      "        [ 0.0139,  1.5501, -1.5565, -0.4320, -0.0849],\n",
      "        [-0.9348, -0.0367, -0.0308, -0.1045, -2.1454]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.6121, 0.6602, 0.7697, 0.7827, 0.0496, 0.9708, 0.1045, 0.6400, 0.3753,\n",
      "        0.0372], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.6765, -0.5366,  0.7537,  0.6973, -0.2330],\n",
      "        [ 0.1544,  1.4454,  0.6061,  2.3954, -2.5873],\n",
      "        [ 2.7417, -0.4849, -0.0958, -0.0505,  0.3022],\n",
      "        [-0.1985, -0.7887, -0.5401, -0.8778,  1.9601],\n",
      "        [-0.1167,  1.9408,  0.8146,  0.1569, -0.0145],\n",
      "        [ 1.5597, -0.0465,  0.5427, -0.3812,  1.4674],\n",
      "        [ 0.8259, -0.2445, -0.6404, -1.7903, -0.1180],\n",
      "        [ 0.3929,  0.0281,  0.4765,  0.5026,  1.8940],\n",
      "        [-0.1203, -1.2754, -0.4173,  0.0278, -0.9836],\n",
      "        [-0.2289, -0.0262, -0.7949, -0.2147,  0.2236]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.5013, 0.8823, 0.9178, 0.3906, 0.9416, 0.9586, 0.1227, 0.9642, 0.0590,\n",
      "        0.2609], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-7.5056e-02,  1.0648e+00,  8.6155e-03,  9.5772e-01,  4.8095e-01],\n",
      "        [-1.9418e-01, -3.2435e+00,  2.7459e-01,  5.8506e-01, -9.8383e-01],\n",
      "        [ 1.7483e-01,  3.9206e-01, -1.0172e+00, -2.0588e-02,  4.1804e-01],\n",
      "        [-2.0776e-01, -1.2749e+00,  8.4765e-01,  5.9372e-03, -1.1343e-01],\n",
      "        [ 1.8984e-01,  7.1749e-01, -1.0869e-01,  6.3966e-01, -4.9746e-01],\n",
      "        [ 6.3136e-01,  1.2607e-02, -7.2067e-01, -3.4206e-01, -5.4162e-01],\n",
      "        [ 3.5858e-01, -1.6102e-01,  5.5180e-01,  5.1301e-01, -2.9719e-01],\n",
      "        [ 4.1400e+00,  8.6431e-01, -4.9873e-01,  1.8644e-03, -7.3528e-01],\n",
      "        [-4.6236e-01,  3.2347e-01,  2.0793e+00,  2.0308e+00, -2.3467e+00],\n",
      "        [-2.7762e-01,  5.9418e-01,  9.0952e-01,  1.1296e+00,  9.5733e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9196, 0.0276, 0.4868, 0.3224, 0.7193, 0.2768, 0.7242, 0.9775, 0.8354,\n",
      "        0.9207], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.1865,  0.1314, -0.5962,  0.1332, -0.0859],\n",
      "        [-0.2556,  1.8064,  0.0800, -0.2424,  0.1827],\n",
      "        [ 0.8831, -0.2498,  0.4181, -0.5802,  0.0419],\n",
      "        [ 0.4004, -1.0362, -0.7483,  1.6833, -0.1856],\n",
      "        [ 0.2250, -0.2452,  0.6559,  1.6589,  0.0187],\n",
      "        [-0.1929, -0.4646,  0.0811,  0.3986,  1.3774],\n",
      "        [ 0.8124,  0.0902,  0.6215, -0.9449,  1.0960],\n",
      "        [ 0.5758,  0.1118,  0.1955, -2.5573,  0.1004],\n",
      "        [ 0.0310,  0.7465,  0.5693, -0.0069, -0.7961],\n",
      "        [-0.0119,  0.6963, -0.0227,  0.3695,  0.2728]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3535, 0.8280, 0.6255, 0.5284, 0.9100, 0.7685, 0.8423, 0.1717, 0.6327,\n",
      "        0.7865], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.1005e-01, -8.6651e-01,  7.6541e-01, -6.2633e-03,  3.7347e-01],\n",
      "        [-1.4947e+00,  1.0413e+00, -5.4566e-01,  8.2120e-01,  6.2651e-01],\n",
      "        [-8.8902e-01, -2.3354e+00, -3.6743e-02,  4.6483e-02,  7.5304e-01],\n",
      "        [ 6.4151e-01,  1.6431e-01, -3.2259e-01,  7.5245e-01,  1.6335e+00],\n",
      "        [-1.8405e-01, -7.0326e-01, -3.8260e-02, -5.0979e-01, -6.4210e-02],\n",
      "        [-3.1088e-01,  1.0073e+00,  3.3142e-01, -2.8854e+00, -1.1809e-02],\n",
      "        [-2.5694e+00,  4.2732e-02,  1.0963e+00,  1.0440e+00,  1.0707e+00],\n",
      "        [-6.0922e-01,  4.5589e-04, -1.5583e-01,  1.4011e-01,  7.3901e-01],\n",
      "        [-6.1453e-01,  2.2917e-01, -2.3798e-01, -9.4559e-02, -1.4841e+00],\n",
      "        [-2.5775e+00,  6.9611e-01, -6.5267e-01,  1.1730e+00,  2.8353e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.5929, 0.6103, 0.0786, 0.9463, 0.1825, 0.1336, 0.6647, 0.5286, 0.0996,\n",
      "        0.2540], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.957368  [  600/  981]\n",
      "out: tensor([[-1.2780e+00, -1.4792e-01,  1.8245e+00,  3.1479e-01,  1.7878e-01],\n",
      "        [-6.4424e-01,  1.0175e-01, -2.5113e-01,  1.2026e-01, -9.0659e-02],\n",
      "        [-2.9413e-01,  1.6708e-02, -1.5442e-01,  4.2435e-01, -2.7724e+00],\n",
      "        [-2.1590e-01, -3.3935e+00, -3.8993e+00,  2.9318e-03,  1.6926e-01],\n",
      "        [ 3.2474e-03, -3.8442e+00,  3.7548e-02,  1.6376e-01, -1.9708e+00],\n",
      "        [ 6.9202e-01, -1.4801e-01, -3.2424e-01,  4.1600e-03, -1.1564e-01],\n",
      "        [-4.0538e-01,  1.8964e-01,  4.0399e-01, -3.9856e-01, -1.4337e+00],\n",
      "        [-4.6984e-01, -1.5270e-01, -1.1293e-01,  8.3439e-01, -3.3722e+00],\n",
      "        [-7.8804e-02,  6.2260e-01,  5.7083e-01, -3.1729e-02, -6.2448e-01],\n",
      "        [-3.1115e-01, -3.7186e-01, -1.1123e+00, -2.3304e-01, -1.3952e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([7.0934e-01, 3.1777e-01, 5.8423e-02, 6.5094e-04, 3.6461e-03, 5.2705e-01,\n",
      "        1.6192e-01, 3.6501e-02, 6.1264e-01, 1.1483e-01],\n",
      "       grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-2.5694,  0.0427,  1.0963,  1.0440,  1.0707],\n",
      "        [-0.1167,  1.9408,  0.8146,  0.1569, -0.0145],\n",
      "        [-1.4745, -1.6744,  0.1682,  0.0665, -4.9866],\n",
      "        [ 0.1559, -1.7486, -0.4097, -0.0264, -1.7835],\n",
      "        [-0.8235, -1.2520, -1.8047, -0.4852, -0.6643],\n",
      "        [-0.4872,  0.8677, -0.7271, -0.2077, -1.7281],\n",
      "        [-0.6485,  0.2612,  1.3738,  2.2970,  0.2351],\n",
      "        [-1.1842,  0.8882, -0.4656, -1.4277, -0.0617],\n",
      "        [-0.2896,  1.1345, -1.1631, -0.1580, -0.0855],\n",
      "        [ 0.5030, -0.0499,  1.2419,  0.5035, -1.1152]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([6.6469e-01, 9.4165e-01, 3.7033e-04, 2.1621e-02, 6.4980e-03, 9.2596e-02,\n",
      "        9.7121e-01, 9.5263e-02, 3.6316e-01, 7.4712e-01],\n",
      "       grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-1.3955, -0.2835,  0.1304,  2.7277,  0.2966],\n",
      "        [ 0.1023,  1.3143,  0.8693,  0.0620,  0.1028],\n",
      "        [-0.5863, -0.5463,  0.1876, -0.1058,  1.1014],\n",
      "        [-0.0171,  0.9132,  0.2292,  0.1096,  0.0955],\n",
      "        [-0.0447,  0.0810, -0.2684,  0.8844, -0.8443],\n",
      "        [-1.2035, -0.0564,  0.2418,  0.6532, -0.1335],\n",
      "        [ 1.2018,  0.2552, -0.1165,  0.7428, -0.0739],\n",
      "        [ 0.8914,  1.6704,  0.0529, -0.5243, -0.4504],\n",
      "        [-0.3703,  0.6299, -0.8164,  0.0093,  0.0906],\n",
      "        [-0.3350,  1.9668, -0.9360,  1.1309,  1.3042]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8139, 0.9206, 0.5126, 0.7909, 0.4521, 0.3779, 0.8818, 0.8375, 0.3877,\n",
      "        0.9581], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.1344, -0.8441,  0.0194, -0.1274,  0.0319],\n",
      "        [ 0.0893, -0.1364, -0.6384,  0.1716, -0.0544],\n",
      "        [-0.1237, -1.1516,  0.9831, -0.7916, -0.0052],\n",
      "        [ 1.4425, -0.1199,  0.0408,  1.0849,  0.4064],\n",
      "        [ 0.5368,  0.1171, -1.0561, -1.0884, -0.0753],\n",
      "        [ 2.5859, -1.6476,  0.8274, -0.0028, -1.9932],\n",
      "        [ 0.2950, -2.7135, -0.1629,  0.3507, -0.0902],\n",
      "        [-1.2390, -0.1651,  0.3528,  0.0564, -0.9022],\n",
      "        [ 0.9745, -0.5556, -0.7098,  0.2748,  1.5539],\n",
      "        [ 0.0140,  0.3369,  0.6951, -0.1060, -1.8272]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3131, 0.3617, 0.2518, 0.9456, 0.1728, 0.4427, 0.0894, 0.1304, 0.8231,\n",
      "        0.2917], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 5.8070e-01, -1.0901e+00,  4.8897e-01,  1.7491e-02,  4.5905e-01],\n",
      "        [ 2.3953e-01,  1.2584e+00, -9.1196e-02, -1.4481e+00, -2.1898e+00],\n",
      "        [ 1.9593e+00, -2.3410e+00,  6.8007e-02,  3.4527e-01, -5.2769e-01],\n",
      "        [ 3.3228e-04, -2.0220e+00, -9.5067e-02,  1.2011e-01,  2.8382e-01],\n",
      "        [ 9.6090e-02,  7.2444e-02, -4.4405e-01,  5.0841e-01, -5.6316e-01],\n",
      "        [-1.1049e-01,  2.3276e+00, -2.3810e-01,  7.7475e-02, -4.1224e-01],\n",
      "        [-2.0825e-01, -5.6398e-01,  2.2177e-01,  5.9777e-01,  9.4139e-02],\n",
      "        [ 8.3266e-01,  5.2693e-01,  1.9621e-01, -1.4666e+00,  2.4209e-01],\n",
      "        [ 1.9533e-01,  2.6360e-01, -1.1442e-01,  1.6727e-01,  9.0817e-02],\n",
      "        [ 3.4775e-01, -4.5240e-01, -1.3419e+00, -2.7563e-01, -6.3610e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.6121, 0.0970, 0.3785, 0.1528, 0.4182, 0.8381, 0.5353, 0.5821, 0.6463,\n",
      "        0.0864], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.693533  [  650/  981]\n",
      "out: tensor([[-1.1497e+00, -2.7062e-02,  9.2816e-02,  5.4761e-01,  7.9370e-02],\n",
      "        [-6.4593e-02,  6.1615e-01, -3.5661e-02,  8.1254e-01,  7.3807e-01],\n",
      "        [-4.2271e-02, -2.6457e+00,  9.3663e-01,  5.3464e-01,  9.8051e-02],\n",
      "        [ 5.4859e-01,  1.0539e-03,  4.2895e-02,  2.6278e-01, -9.9998e-01],\n",
      "        [ 6.4149e-01,  2.5164e+00, -3.0942e-01, -2.9262e+00, -2.0791e+00],\n",
      "        [-7.5597e-01,  4.1079e-01,  7.5532e-02,  5.6927e-02,  3.8736e-01],\n",
      "        [ 5.6423e-01, -4.5180e-01,  7.9544e-01,  1.4288e-01,  3.8190e-01],\n",
      "        [ 1.2489e-01, -5.6246e-01,  9.9608e-01, -5.5498e-01,  1.5252e-02],\n",
      "        [ 2.6636e+00, -5.7470e-01,  2.1125e+00, -3.1208e-01, -4.5581e-01],\n",
      "        [ 1.0199e-01,  3.2546e-01,  5.0179e-01, -1.3708e-01, -1.9252e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3877, 0.8876, 0.2463, 0.4639, 0.1037, 0.5435, 0.8073, 0.5047, 0.9687,\n",
      "        0.2436], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-2.9711, -0.4023, -1.0208,  0.4086, -0.4096],\n",
      "        [-0.4252, -1.6492,  0.5818, -0.2453,  0.0861],\n",
      "        [-0.5988,  2.2654, -0.0954,  0.0928, -0.1330],\n",
      "        [-0.2717, -0.6809, -1.2670, -1.5533,  0.2408],\n",
      "        [ 0.0209, -0.0791, -1.2035,  0.2077, -0.8268],\n",
      "        [ 0.2424, -0.6220, -1.0752,  0.0352,  0.1652],\n",
      "        [-0.3667, -0.2895,  0.4790,  0.4868,  0.2757],\n",
      "        [ 0.3243,  2.1518, -0.0329,  0.1269, -1.6721],\n",
      "        [-0.3790, -0.0083,  0.4111,  0.1933, -1.3414],\n",
      "        [-1.4843, -1.5826,  0.7825, -0.2922,  3.1930]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0122, 0.1609, 0.8222, 0.0284, 0.1323, 0.2219, 0.6423, 0.7106, 0.2452,\n",
      "        0.6494], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-3.1076e-01, -9.9974e-01, -4.6750e-01, -5.0301e+00, -4.7570e-02],\n",
      "        [-9.5576e-02, -5.5941e-01, -8.7454e-02,  1.3739e+00, -1.9254e-01],\n",
      "        [ 2.4594e-01,  3.8098e+00, -6.0701e-01, -6.7237e-03,  5.1333e-01],\n",
      "        [-1.0367e-01,  2.1835e-01,  1.9106e+00,  2.0808e-02,  2.3416e-01],\n",
      "        [-5.2876e+00, -9.1827e-02, -3.6073e-02,  1.5729e-01, -7.7267e-02],\n",
      "        [ 3.0137e-01, -2.6676e-01,  3.1269e-01, -1.5465e+00, -1.0345e-01],\n",
      "        [-2.4741e-01,  2.2188e-01, -8.7769e-02,  2.4660e-03, -7.8774e-02],\n",
      "        [ 3.8956e-01,  8.9459e-01, -3.7519e+00,  1.6492e+00,  2.4910e-01],\n",
      "        [-6.0819e-01,  9.1362e-02,  4.6766e-01, -9.1610e-01, -7.5874e-01],\n",
      "        [-2.6768e-01, -1.3187e-01,  5.5163e-01,  4.8269e-01,  1.1366e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0011, 0.6080, 0.9812, 0.9072, 0.0048, 0.2137, 0.4527, 0.3614, 0.1514,\n",
      "        0.6788], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.0615, -0.3336, -1.5807, -0.4669,  0.1752],\n",
      "        [-0.3395, -1.2824,  1.0757,  0.3373, -0.0948],\n",
      "        [ 0.1248,  0.3777, -0.1503,  0.2456,  0.6633],\n",
      "        [ 0.2637,  0.0410, -0.1546, -0.0076,  1.3875],\n",
      "        [ 0.9381, -0.8453,  0.2638, -0.1056, -1.1428],\n",
      "        [-0.2321, -0.2312,  0.4691, -3.7626,  0.0124],\n",
      "        [-0.0206,  3.0228, -0.2359,  0.1097,  1.3018],\n",
      "        [ 0.6583, -0.0840, -0.6453,  0.3672,  0.6559],\n",
      "        [ 0.1668,  0.0441, -0.3196, -0.2514, -0.5938],\n",
      "        [-1.5540,  1.4942, -0.0503,  0.6466,  0.4948]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1048, 0.4246, 0.7792, 0.8220, 0.2907, 0.0231, 0.9849, 0.7215, 0.2781,\n",
      "        0.7372], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 3.6324e-01,  3.7166e-01,  1.3153e+00, -1.1930e+00,  1.0001e-01],\n",
      "        [-3.8590e-02, -3.1735e-01, -4.4372e-01,  3.8982e+00,  2.3127e-01],\n",
      "        [ 7.7246e-01, -8.4544e-01,  3.9923e+00,  9.1785e-03,  8.5176e-02],\n",
      "        [-1.2023e-02,  8.0417e-01,  1.4382e-01, -4.0498e-01, -6.9800e-01],\n",
      "        [-8.9309e-02, -2.2480e+00, -4.0524e-01, -1.9552e-01, -5.8908e-02],\n",
      "        [ 7.4271e-04, -1.4562e+00,  5.3126e-01,  1.2049e+00,  7.5003e-01],\n",
      "        [-1.0880e+00,  1.1925e-01, -3.9598e-01, -5.1023e-01, -2.0767e-02],\n",
      "        [-9.5902e-01,  3.3985e-01,  1.8228e+00,  1.7594e+00, -4.0855e-02],\n",
      "        [ 2.9731e+00, -1.5389e-01, -1.2845e-01,  1.9541e-01, -8.8218e-01],\n",
      "        [ 3.0171e-01,  5.5751e-01,  2.9520e+00, -8.5517e-02,  9.3635e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7226, 0.9654, 0.9823, 0.4583, 0.0476, 0.7371, 0.1306, 0.9489, 0.8812,\n",
      "        0.9906], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.772529  [  700/  981]\n",
      "out: tensor([[ 5.2083e-01,  1.4139e+00,  2.0840e-01, -1.8324e-01,  8.9163e-02],\n",
      "        [ 5.1191e-01, -6.6476e-01, -5.4395e-01, -4.0702e+00, -7.5996e-01],\n",
      "        [ 5.3754e-01,  5.5876e-02,  7.2043e-02, -1.1586e-01,  1.5011e+00],\n",
      "        [ 3.5378e-01, -4.4855e-02,  3.7487e-02, -6.8988e-01, -7.7922e-01],\n",
      "        [ 5.3510e-01, -1.1912e+00, -3.2620e-02, -2.9570e-01,  3.5492e-01],\n",
      "        [-3.3579e-01,  3.2569e+00,  6.6118e-01, -1.6253e-02,  1.2671e+00],\n",
      "        [ 7.3312e-04,  1.9099e-01,  6.7198e-01,  1.4748e+00, -1.1205e+00],\n",
      "        [ 5.9242e-02,  1.3147e+00, -4.2796e-01,  2.9077e-01,  8.9150e-01],\n",
      "        [-7.1743e-01, -1.0515e+00, -1.1511e+00,  8.0657e-01, -2.2777e+00],\n",
      "        [-8.3299e-02,  3.0131e+00, -8.6350e-03,  1.3944e+00, -2.9644e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8858, 0.0040, 0.8860, 0.2455, 0.3476, 0.9921, 0.7717, 0.8936, 0.0122,\n",
      "        0.9823], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.1884, -1.0319, -0.4784, -0.3771, -0.1718],\n",
      "        [-3.3995, -1.4551, -2.9629, -0.3176,  0.2476],\n",
      "        [ 1.4704,  1.9627, -0.1517, -1.6047,  0.8883],\n",
      "        [ 0.2517, -1.7325,  1.4762, -0.4466,  0.1174],\n",
      "        [-0.0718,  0.5319, -0.7394,  0.0601, -1.0149],\n",
      "        [ 0.3466, -3.7549,  0.4827,  0.1303,  0.5156],\n",
      "        [ 0.7320, -0.1616,  2.8794, -0.0987,  0.0297],\n",
      "        [ 0.1720, -0.0141,  0.0434, -0.5446, -2.1158],\n",
      "        [ 0.3199,  1.7391, -0.8288,  0.1947, -0.0366],\n",
      "        [-0.6606,  0.0180,  0.3671, -0.3767, -1.3680]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([9.5563e-02, 3.7529e-04, 9.2858e-01, 4.1729e-01, 2.2547e-01, 9.2818e-02,\n",
      "        9.6710e-01, 7.8780e-02, 8.0030e-01, 1.1709e-01],\n",
      "       grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.1352,  1.2701, -1.0448, -0.4436, -0.0729],\n",
      "        [ 0.8226,  1.1410, -0.2101, -1.2023, -0.6219],\n",
      "        [-0.2523, -0.7082,  1.1862, -1.3243, -0.8886],\n",
      "        [-0.7427, -1.7035,  0.7794,  2.6725, -0.1352],\n",
      "        [ 0.5981,  2.2770, -1.6452,  0.2518, -0.6835],\n",
      "        [ 0.0208,  0.4557, -0.6889,  0.0362,  0.0323],\n",
      "        [-0.2310, -0.8834,  0.0122,  1.0345,  0.6862],\n",
      "        [ 0.3498,  0.4710, -0.3002, -0.0224,  0.8030],\n",
      "        [ 0.5640,  0.8591, -0.5228, -1.3802,  0.1920],\n",
      "        [-0.8051,  0.9088,  0.5532, -0.0278,  1.5129]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3950, 0.4823, 0.1206, 0.7049, 0.6896, 0.4641, 0.6499, 0.7860, 0.4285,\n",
      "        0.8949], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.3791e-01, -2.2024e-02, -1.9116e+00,  6.6566e-01, -6.9990e-01],\n",
      "        [-4.2890e-01,  2.7292e+00, -2.7750e-02,  1.2231e+00,  1.4359e-01],\n",
      "        [ 3.0912e-01, -2.1974e-01,  2.9858e+00, -1.1864e-01,  8.6269e-02],\n",
      "        [ 2.6821e-02,  1.6409e-02,  9.4107e-01, -3.3293e-01, -6.3224e-02],\n",
      "        [-6.3877e-01,  2.8066e+00,  1.0449e+00,  1.6853e+00,  1.9709e+00],\n",
      "        [ 1.3441e+00, -1.7537e+00, -8.9212e-03, -6.9745e-02, -9.3775e-01],\n",
      "        [-2.9107e-01, -2.1293e-03, -1.9245e-01, -1.4988e-01, -1.3852e+00],\n",
      "        [ 4.3568e-02, -4.1924e-01,  6.7471e-02, -3.4845e-01, -1.1805e-01],\n",
      "        [-2.9412e-01,  1.6717e-02, -1.5442e-01,  4.2434e-01, -2.7723e+00],\n",
      "        [-8.4114e-01,  1.0585e+00, -3.5604e-01, -1.7659e+00,  2.4158e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1382, 0.9744, 0.9545, 0.6429, 0.9990, 0.1937, 0.1170, 0.3155, 0.0584,\n",
      "        0.6251], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.2433e-01,  6.0297e-02, -4.2634e-01, -1.8519e-01,  1.7203e+00],\n",
      "        [-4.6487e-02, -1.5570e+00, -7.6268e-01,  3.0309e-01, -6.3002e-01],\n",
      "        [ 3.0194e-01,  3.6791e-01,  4.6685e-01,  2.2664e+00,  7.3812e-02],\n",
      "        [-5.1280e-01,  1.6864e-04,  7.1186e-02, -1.2438e-01,  2.5418e-01],\n",
      "        [-2.4129e-01, -1.6579e-03, -1.1836e-01, -1.3713e-01, -4.7420e-01],\n",
      "        [ 1.2266e+00, -3.3320e-01,  1.1119e+00,  9.5663e-01, -2.6024e+00],\n",
      "        [ 7.7044e-01, -5.4205e-02,  7.2748e-02, -2.8083e+00, -5.9326e-01],\n",
      "        [ 4.7843e-01, -2.7232e-01, -2.1251e-01,  8.0228e-01, -6.6666e-02],\n",
      "        [ 7.3758e-01,  3.5928e-01,  8.7487e-01,  2.0449e-01, -9.5684e-01],\n",
      "        [ 6.2502e-03, -1.2242e-01,  3.1025e-01, -1.8004e-02, -7.8054e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7847, 0.0634, 0.9700, 0.4227, 0.2744, 0.5889, 0.0683, 0.6746, 0.7720,\n",
      "        0.3533], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.454230  [  750/  981]\n",
      "out: tensor([[ 1.5401e-04, -2.7298e-01, -1.6346e+00, -1.7018e+00,  6.2325e-02],\n",
      "        [-4.6831e-01,  1.1410e-01,  2.8351e-01,  8.4634e-01, -7.8791e-01],\n",
      "        [-2.3680e-02, -8.4888e-01, -3.5796e-01, -2.8371e-01,  6.6303e-01],\n",
      "        [ 4.3041e-01,  3.7622e-01,  4.8720e-01, -1.5227e+00, -1.1370e+00],\n",
      "        [-2.6724e-01, -1.7580e+00, -1.4324e-01, -5.3887e-01,  1.3517e-01],\n",
      "        [-7.7489e-02,  6.9312e-01, -8.5723e-01,  1.1267e-01,  5.8451e-01],\n",
      "        [-3.0040e-04,  9.7601e-01, -3.2487e-01,  7.6861e-01, -7.6688e-01],\n",
      "        [ 2.0014e-01, -4.6490e-03,  5.6993e-01, -2.1261e-01, -7.0659e-01],\n",
      "        [ 5.4362e-02, -1.3440e+00,  3.6938e-01,  5.5575e-01, -2.5635e-01],\n",
      "        [ 2.4313e+00,  4.9088e-03, -1.9737e+00,  3.1301e-01,  3.9236e-02]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.0280, 0.4969, 0.2992, 0.2033, 0.0710, 0.6120, 0.6576, 0.4616, 0.3496,\n",
      "        0.6931], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.2252, -1.1209,  0.5254,  0.2588, -0.2834],\n",
      "        [ 0.0879,  0.0195,  1.1317,  0.1753, -0.2012],\n",
      "        [-1.5858,  0.9753, -1.0873, -0.4692,  0.3033],\n",
      "        [-0.9465, -0.5352, -0.4159,  0.0053, -0.7950],\n",
      "        [ 0.1788, -0.1793,  0.1427, -0.0195, -2.0960],\n",
      "        [-0.2717, -0.6809, -1.2670, -1.5533,  0.2408],\n",
      "        [ 0.0566,  0.4466,  1.8005, -0.1148,  0.1109],\n",
      "        [-1.2470,  0.1343, -0.5433, -2.7340, -0.1953],\n",
      "        [ 0.0224, -1.7982,  0.8458, -0.5641, -0.3228],\n",
      "        [-1.5530,  0.0181, -0.1045, -0.2801, -0.3797]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.4025, 0.7708, 0.1343, 0.0637, 0.1220, 0.0284, 0.9089, 0.0101, 0.1398,\n",
      "        0.0912], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 5.1102e-02,  2.9528e-01, -2.2183e-01, -5.9552e-01, -6.4382e-01],\n",
      "        [ 7.3199e-01,  1.3446e+00,  1.2356e-01,  1.0599e+00,  3.1970e-01],\n",
      "        [-1.0426e-01, -8.6748e-01, -8.3828e-02,  2.7204e+00,  8.9632e-02],\n",
      "        [-1.0274e+00, -1.2835e+00, -3.6963e-02,  1.7578e-01,  1.8405e-01],\n",
      "        [-9.7534e-01, -8.4618e-02,  2.0656e-01, -4.0953e-02, -1.5934e+00],\n",
      "        [-7.4463e-01,  2.5177e-01,  2.0684e-01,  2.6522e-01,  1.3438e-02],\n",
      "        [-6.5552e-02,  7.5319e-02,  1.4782e+00,  5.2573e-01, -9.2778e-01],\n",
      "        [-4.7892e-04, -4.0570e-01, -1.3594e+00, -4.4361e-02,  7.0202e-01],\n",
      "        [ 4.2529e-04,  1.8144e-01,  2.2424e+00,  1.3444e+00, -9.9636e-02],\n",
      "        [ 9.7449e-01, -5.5562e-01, -7.0979e-01,  2.7474e-01,  1.5538e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.2470, 0.9729, 0.8525, 0.1205, 0.0767, 0.4982, 0.7476, 0.2483, 0.9751,\n",
      "        0.8231], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 2.6070,  0.3641,  1.7067,  0.5413,  0.1986],\n",
      "        [ 0.0302,  2.2365,  1.0547,  1.2925, -1.4179],\n",
      "        [ 0.6832, -0.0396, -0.4963,  1.1268, -0.0171],\n",
      "        [ 0.0293, -2.1896, -1.3177,  0.0130, -0.5357],\n",
      "        [ 0.6735,  0.0095, -0.1257, -0.0397, -1.1042],\n",
      "        [ 1.1348,  1.4406, -0.0592,  0.2730,  0.6460],\n",
      "        [ 0.5352, -0.3457, -0.1911, -1.1258,  0.5242],\n",
      "        [ 0.3843, -1.0075,  0.1196,  2.0151, -0.6572],\n",
      "        [ 0.0298, -2.2897,  0.4071, -0.1039,  1.2286],\n",
      "        [ 0.2773,  0.0293, -0.0478,  0.3418,  0.1394]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9956, 0.9607, 0.7785, 0.0180, 0.3574, 0.9688, 0.3536, 0.7015, 0.3256,\n",
      "        0.6770], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.3605, -0.4004,  0.4676, -0.1467,  1.0879],\n",
      "        [-0.0317,  1.3540, -1.3502,  0.4229,  0.0826],\n",
      "        [ 0.3252,  0.3529, -0.4563,  0.0543,  0.0710],\n",
      "        [-1.5256, -0.0879, -0.3470,  0.9208,  0.2663],\n",
      "        [-0.7443,  0.2500,  0.2142,  0.2233,  0.3362],\n",
      "        [ 0.5739, -0.0224, -0.0690,  0.3168,  0.4007],\n",
      "        [ 0.0725, -0.2262,  0.4968, -0.1407, -0.0483],\n",
      "        [-0.4980, -0.0158,  0.1080, -0.0225, -1.4993],\n",
      "        [-0.0061,  1.8787, -0.5221,  0.2580,  0.5628],\n",
      "        [ 2.0673,  0.0345,  0.2620,  0.1677,  0.8811]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9144, 0.6172, 0.5859, 0.3157, 0.5694, 0.7685, 0.5385, 0.1270, 0.8976,\n",
      "        0.9681], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.496008  [  800/  981]\n",
      "out: tensor([[ 0.3417, -0.0927, -0.7355, -0.8667,  2.7533],\n",
      "        [ 0.0487,  1.5638,  0.0037, -0.2384,  2.1606],\n",
      "        [-0.7500,  0.0786, -0.2725,  0.7649,  0.1010],\n",
      "        [ 0.2826, -0.6437, -0.1936, -1.8006,  1.2843],\n",
      "        [-0.3111, -0.3718, -1.1122, -0.2330, -0.0139],\n",
      "        [ 0.2019,  0.0786, -0.0646, -0.1803, -0.5285],\n",
      "        [ 0.0130,  0.2283,  0.1086,  0.0884,  0.5793],\n",
      "        [ 0.2236,  0.4886, -1.7012,  0.7355, -0.1520],\n",
      "        [-0.0925, -0.6988, -0.4080, -0.3618, -0.1263],\n",
      "        [-0.0538,  0.0093,  0.6255,  0.8004, -0.1389]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8022, 0.9718, 0.4805, 0.2552, 0.1148, 0.3792, 0.7345, 0.4000, 0.1561,\n",
      "        0.7760], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.3288, -0.2817,  1.3467, -0.2817, -0.0129],\n",
      "        [-0.1865, -1.2452,  1.9773, -0.0311, -0.0792],\n",
      "        [ 0.7275,  0.0689,  1.0906, -0.4256,  0.2047],\n",
      "        [ 0.5982,  0.9047,  0.1651, -0.7805,  0.6295],\n",
      "        [ 0.5923, -0.7738, -0.6609, -0.0107,  2.2009],\n",
      "        [ 0.0512, -0.1632,  0.3041, -0.0090, -0.9509],\n",
      "        [-1.4398, -0.0953,  0.6440, -0.2973,  1.5929],\n",
      "        [ 0.0097, -0.8041,  0.3985,  0.5798, -0.0125],\n",
      "        [ 1.2697, -0.0874, -4.8976,  0.1603, -1.1055],\n",
      "        [ 0.4216,  0.4494,  0.2017, -0.1703,  0.2866]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7501, 0.6071, 0.8410, 0.8201, 0.7938, 0.3170, 0.5998, 0.5427, 0.0094,\n",
      "        0.7666], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.2607,  0.2071, -0.1276,  0.4960,  0.4855],\n",
      "        [-0.0056,  1.8711, -0.7500,  0.0771,  0.0638],\n",
      "        [ 0.4686,  0.5448, -0.2068, -0.1567,  1.2772],\n",
      "        [-0.0067, -0.4968,  1.1094, -0.0477, -0.4255],\n",
      "        [ 0.0217, -0.0251, -0.5024,  0.4861, -1.3700],\n",
      "        [-0.6220, -1.1494, -0.9506,  0.4528,  2.5951],\n",
      "        [-1.4535,  1.1707, -0.8507, -0.1958, -0.5303],\n",
      "        [ 1.2128, -0.6683,  1.6143, -0.7765,  0.7553],\n",
      "        [ 0.2173, -1.3709, -0.4742,  0.0629, -0.2781],\n",
      "        [ 1.0336, -0.6891, -2.4214,  0.6033, -1.5043]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7895, 0.7784, 0.8729, 0.5331, 0.1994, 0.5808, 0.1348, 0.8945, 0.1367,\n",
      "        0.0484], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 2.2498e-01, -2.5458e-02,  1.5133e-01,  1.2931e-01,  1.6923e+00],\n",
      "        [-2.8073e-01,  4.2195e-01, -1.1057e+00, -4.7862e-03, -5.9473e-01],\n",
      "        [-3.7869e-01,  3.2489e-01,  2.0941e-01,  1.6148e-01, -8.4021e-02],\n",
      "        [-7.5831e-01,  1.6445e+00, -2.2855e+00, -7.2019e-02,  1.2335e+00],\n",
      "        [ 5.6988e-04,  2.4803e-01,  1.7441e+00,  4.3809e-01,  7.2102e-01],\n",
      "        [ 2.1170e-01,  1.6350e+00, -2.4784e-02, -2.3693e-01, -1.4200e-02],\n",
      "        [ 1.2299e-01,  6.4429e-01, -1.2201e+00,  3.4364e-01,  1.8520e+00],\n",
      "        [-1.1779e+00, -1.5543e+00,  1.4292e+00, -1.0178e-01, -5.0722e-02],\n",
      "        [ 2.7593e-01, -1.4747e-01,  5.5070e-01, -1.5992e-01, -3.2181e-02],\n",
      "        [-6.9183e-01, -3.8571e-03,  1.9472e+00, -7.3813e-01,  2.2029e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8978, 0.1731, 0.5580, 0.4408, 0.9590, 0.8279, 0.8510, 0.1892, 0.6194,\n",
      "        0.9380], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.1811,  0.0080, -0.5651, -0.4981, -0.8585],\n",
      "        [ 0.4047,  0.6512,  0.1037, -1.2110,  0.0194],\n",
      "        [-0.0796,  0.7301,  1.6204, -0.0280,  0.7712],\n",
      "        [-0.4872,  0.8678, -0.7270, -0.2077, -1.7281],\n",
      "        [ 0.1462,  3.5717,  0.0132,  1.0159, -0.2289],\n",
      "        [ 0.3515,  0.5116,  0.2866,  0.2262, -3.1813],\n",
      "        [-0.4423,  1.1947,  0.0061,  0.2813,  0.1095],\n",
      "        [ 0.1205, -1.6058, -0.7303, -0.9655, -1.9025],\n",
      "        [-0.8450,  0.0507, -0.0962, -0.7224,  0.8999],\n",
      "        [-0.0104,  1.2329, -0.2835, -0.0208, -0.5009]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1096, 0.4920, 0.9532, 0.0926, 0.9892, 0.1412, 0.7594, 0.0062, 0.3289,\n",
      "        0.6029], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.618727  [  850/  981]\n",
      "out: tensor([[-0.1704,  1.3489,  1.0622, -0.1496, -0.8889],\n",
      "        [ 0.5130, -0.7677, -0.9768,  0.2520, -0.0594],\n",
      "        [-0.3059, -1.7919, -0.6472, -2.1674, -0.3270],\n",
      "        [-0.5044,  0.0819, -0.3062, -1.3678,  0.1245],\n",
      "        [ 1.6177,  0.3284,  1.4238, -0.4969, -0.3326],\n",
      "        [ 0.3981, -3.9212, -0.4031,  0.6592,  1.1442],\n",
      "        [-0.2042,  0.0180,  0.7901,  0.1284,  0.1235],\n",
      "        [ 0.6112, -0.0492,  0.6541,  0.4803,  0.9149],\n",
      "        [-0.5284, -1.7912,  1.3145, -0.4620, -0.0377],\n",
      "        [ 2.2744,  1.0409, -0.0053,  0.2701,  0.2067]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.7689, 0.2614, 0.0053, 0.1222, 0.9269, 0.1069, 0.7018, 0.9316, 0.1817,\n",
      "        0.9778], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 4.3937e-01, -3.8397e-01,  8.4898e-02, -9.4026e-01,  4.6238e-02],\n",
      "        [-4.2154e-01, -7.8660e-01, -6.7098e-01,  7.1158e-01, -9.9772e-01],\n",
      "        [-1.2956e-01,  2.8371e+00, -4.5300e-01,  3.8875e-01, -2.4831e-01],\n",
      "        [-2.6329e-01,  3.2508e-03, -2.2820e-01,  1.8307e-01,  2.6641e-01],\n",
      "        [-2.0260e-01, -3.4545e-01,  3.6827e+00,  1.1434e-01,  6.8936e-02],\n",
      "        [ 8.5738e-02,  3.9609e-01, -1.5557e-01, -2.9207e-03,  3.3083e-01],\n",
      "        [-2.4609e-02,  6.4632e-01, -1.2715e-01,  1.3522e+00,  7.2711e-02],\n",
      "        [-9.4564e-01, -1.4828e+00, -3.3481e-02, -1.1640e-02,  2.9042e-01],\n",
      "        [ 5.9913e-02, -4.6084e-01, -3.4347e-01, -1.0068e+00,  8.2302e-01],\n",
      "        [ 6.7412e-01, -2.2243e-02,  9.2612e-02, -4.1517e-01, -8.3780e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3200, 0.1029, 0.9164, 0.4903, 0.9650, 0.6579, 0.8721, 0.1013, 0.2833,\n",
      "        0.3755], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.7939,  0.3488,  1.4184,  0.1391, -0.3062],\n",
      "        [-2.5655,  0.1627,  2.7442,  0.4763,  0.3877],\n",
      "        [-0.9190,  0.4010,  0.2375,  0.9574,  1.2918],\n",
      "        [-3.1609,  0.1341, -0.9165, -0.1035,  0.0080],\n",
      "        [-0.0724,  1.2405, -1.9899, -0.0494, -0.1200],\n",
      "        [-0.0356, -3.4688,  0.1136,  0.0691, -0.4613],\n",
      "        [ 0.4195, -0.0253, -1.0312, -0.1821,  0.8833],\n",
      "        [ 0.9697,  0.2043, -0.7795,  0.6133, -0.1263],\n",
      "        [-0.0343,  4.0312, -0.0198, -1.4152, -0.3733],\n",
      "        [ 0.8141, -0.1453, -0.2619,  0.4246, -0.3108]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.6913, 0.7695, 0.8775, 0.0173, 0.2707, 0.0222, 0.5161, 0.7071, 0.8992,\n",
      "        0.6273], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.9738e+00, -8.3455e-02,  1.6390e-01,  1.6533e-03,  1.2025e-01],\n",
      "        [ 1.8752e+00,  6.9556e-01,  8.4482e-02, -6.6445e-01, -3.4169e-01],\n",
      "        [ 7.1010e-02,  1.0018e+00, -8.9446e-01, -8.0044e-01, -1.5489e-01],\n",
      "        [-6.6013e-01,  4.6217e-01,  1.0831e-02,  7.3191e-01,  7.8996e-01],\n",
      "        [-1.0715e-01, -9.9800e-02,  4.9596e-01,  2.4557e+00, -2.9406e+00],\n",
      "        [-8.2706e-02, -1.8518e+00, -1.4828e-01, -1.3036e-01, -8.6195e-01],\n",
      "        [ 1.4392e+00, -6.7778e-01, -2.2671e-01, -2.4826e+00,  9.3609e-01],\n",
      "        [ 1.9804e-01,  1.9802e-01, -1.5225e-01, -3.6457e-01, -6.2338e-01],\n",
      "        [-1.4885e+00, -2.2490e-02, -2.3166e+00,  1.0263e+00,  3.2769e-01],\n",
      "        [-3.1841e-01,  1.3810e+00,  5.5767e-01, -1.3422e+00, -5.3994e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.8981, 0.8388, 0.3150, 0.7916, 0.4512, 0.0441, 0.2666, 0.3221, 0.0777,\n",
      "        0.4349], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 0.7275, -1.3063, -0.1566,  0.0412,  0.1992],\n",
      "        [ 0.9981,  1.2628,  0.2596,  0.5078, -3.3055],\n",
      "        [ 0.2972,  0.8081,  0.6929, -0.1619,  0.3548],\n",
      "        [-0.0117, -0.4849,  0.5468, -0.0316,  0.4141],\n",
      "        [ 1.3070,  0.8751,  0.4519, -0.3355,  0.5543],\n",
      "        [-0.2077, -1.2749,  0.8476,  0.0059, -0.1134],\n",
      "        [ 0.1023,  0.0052, -0.1902, -0.2481, -0.9451],\n",
      "        [ 1.0226, -0.7502, -0.7412,  0.9862, -0.1403],\n",
      "        [ 0.0884,  0.9328,  0.0545, -0.4775, -0.0051],\n",
      "        [-1.9937,  1.1696, -0.1595,  0.5065,  0.4660]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3787, 0.4311, 0.8799, 0.6065, 0.9455, 0.3224, 0.2182, 0.5932, 0.6441,\n",
      "        0.4972], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.103962  [  900/  981]\n",
      "out: tensor([[-0.4365,  0.8126, -0.5713, -0.8339,  0.3149],\n",
      "        [-1.2856,  1.0131,  0.1503,  0.4056,  0.9160],\n",
      "        [ 0.1491,  0.7402,  0.0363,  1.0098, -0.6426],\n",
      "        [-0.0204,  0.9475, -0.7553,  1.2443, -1.1467],\n",
      "        [-0.2938,  1.9452, -1.5419, -0.5632,  0.1180],\n",
      "        [-3.3474,  0.0850, -0.1543, -0.3654,  0.5017],\n",
      "        [ 0.1289, -0.2418, -0.0201, -0.8334,  1.4358],\n",
      "        [-0.0102,  0.1113,  0.2379, -0.0247,  0.8243],\n",
      "        [ 0.0353, -0.8492, -0.3766, -1.4357,  0.0752],\n",
      "        [ 1.2287, -0.0719,  0.2180, -0.3975,  1.1960]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3287, 0.7684, 0.7846, 0.5670, 0.4169, 0.0362, 0.6152, 0.7574, 0.0724,\n",
      "        0.8978], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 2.8326e-01, -4.3194e-01,  3.6395e+00,  2.6812e-02, -6.6838e-01],\n",
      "        [-1.2996e-01, -6.6455e-01, -2.4190e-01,  2.0927e+00, -4.5286e-01],\n",
      "        [-8.8329e-01, -2.2525e-01, -1.7690e-01,  7.7469e-01,  1.8371e+00],\n",
      "        [-7.9142e-02, -1.7388e+00,  9.1006e-01, -1.0062e-02, -9.1115e-01],\n",
      "        [-3.6837e-01,  1.7692e-01, -3.3785e-02,  1.9505e+00,  2.1570e-03],\n",
      "        [-2.0873e-02, -7.9787e-01, -6.2104e-01,  3.2042e+00,  1.2874e+00],\n",
      "        [-3.4308e-02,  4.0312e+00, -1.9765e-02, -1.4152e+00, -3.7334e-01],\n",
      "        [-7.6364e-01,  1.2455e+00,  1.5748e+00, -6.7525e-01,  2.5989e+00],\n",
      "        [-5.8532e-02, -2.7554e-01,  1.6900e-01,  1.3391e-01,  6.0294e-03],\n",
      "        [-7.2604e-01,  5.0342e-01,  5.3350e-02,  1.8936e+00, -3.3260e-03]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.9453, 0.6464, 0.7902, 0.1383, 0.8491, 0.9549, 0.8992, 0.9817, 0.4937,\n",
      "        0.8483], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-9.7369e-01,  3.5770e-01, -1.3093e+00,  1.2012e+00,  3.2362e-01],\n",
      "        [ 1.0604e+00, -2.0248e-01,  3.4021e+00,  3.4657e-02, -2.2422e-01],\n",
      "        [-2.9147e-01, -1.8233e-01, -6.3977e-01, -3.6203e-01, -9.2315e-02],\n",
      "        [-6.7008e-02,  1.0219e+00, -7.2816e-01, -3.2966e-02, -1.7560e+00],\n",
      "        [-1.0632e+00,  2.1243e-02, -1.2300e+00, -8.3812e-01,  1.6314e+00],\n",
      "        [ 3.4050e-02,  3.5788e-01,  1.9091e+00,  1.1330e+00, -6.0401e-01],\n",
      "        [ 1.0300e+00,  1.0927e-01,  6.4026e-01, -2.9958e-01,  3.6291e+00],\n",
      "        [-2.4367e-03,  1.1275e+00, -3.8476e-01,  6.6097e-01, -8.9366e-01],\n",
      "        [ 3.7677e-01, -1.1458e-01,  1.3033e+00,  1.3645e-02,  1.8176e+00],\n",
      "        [-1.0612e+00,  1.9426e+00,  1.0329e+00, -6.2854e-03, -1.7301e+00]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.4012, 0.9832, 0.1725, 0.1733, 0.1856, 0.9443, 0.9940, 0.6242, 0.9676,\n",
      "        0.5444], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-4.8809e-01,  1.3464e+00, -1.5242e+00,  2.0987e-03, -1.0357e-01],\n",
      "        [-1.8559e-01,  1.1811e-01,  1.3366e+00,  1.4246e-01, -1.6474e+00],\n",
      "        [-1.8346e-01,  8.0771e-01, -4.9137e-02,  2.5658e-01, -9.9777e-03],\n",
      "        [-3.9021e-01, -2.8170e-02, -4.8270e-01, -1.2449e-01,  4.5784e-02],\n",
      "        [ 6.1610e-01, -2.1435e-03, -1.5223e-01,  4.6311e-01,  1.1405e+00],\n",
      "        [-3.2967e+00,  1.4059e+00, -3.3233e-02,  1.3797e-01,  6.9180e-01],\n",
      "        [ 3.7728e-02,  6.2801e-01,  1.4684e-01,  1.7395e-01,  3.3567e-01],\n",
      "        [-4.4523e-02,  1.2876e+00, -8.0057e-01, -2.4462e+00,  2.9605e-01],\n",
      "        [-4.2665e-01, -1.0292e+00, -2.1764e+00, -1.7041e+00, -1.3670e+00],\n",
      "        [ 1.7854e+00, -4.7948e-01,  2.1781e+00,  1.0676e-01,  1.1957e-01]],\n",
      "       grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.3170, 0.4413, 0.6946, 0.2729, 0.8875, 0.2508, 0.7895, 0.1535, 0.0012,\n",
      "        0.9761], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-1.7122,  0.1032, -0.5487, -0.0801,  0.1343],\n",
      "        [-0.4872,  0.8678, -0.7270, -0.2077, -1.7280],\n",
      "        [-0.7426, -1.7034,  0.7794,  2.6725, -0.1352],\n",
      "        [-1.0566,  0.1886,  1.1258, -0.7187,  0.1781],\n",
      "        [ 1.0337, -0.6891, -2.4214,  0.6034, -1.5042],\n",
      "        [ 0.0513, -0.6062, -2.0159,  1.1620, -0.9765],\n",
      "        [-1.0068,  0.9477,  0.6250, -0.4842,  1.4614],\n",
      "        [ 0.1658,  0.2230,  0.8972, -0.0110,  0.3749],\n",
      "        [-2.5626,  0.3140,  0.3617,  0.2824,  1.3587],\n",
      "        [ 0.4658,  0.9909, -0.1198, -1.1381, -0.5498]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.1088, 0.0926, 0.7049, 0.4298, 0.0484, 0.0843, 0.8239, 0.8389, 0.4389,\n",
      "        0.4132], grad_fn=<SigmoidBackward>)\n",
      "loss: 1.098462  [  950/  981]\n",
      "out: tensor([[-0.0387, -0.7671,  1.5118, -1.1272,  1.0077],\n",
      "        [-0.3627,  1.9804, -0.0409, -1.3543,  0.0259],\n",
      "        [-0.5138, -2.0138,  2.0493,  0.1101,  0.0319],\n",
      "        [ 0.0392,  1.4225,  0.7028,  0.0108, -0.0839],\n",
      "        [ 0.5524,  0.2140,  1.4482,  0.6761,  0.8105],\n",
      "        [ 0.5385,  0.9511, -1.6363,  0.1020,  0.2492],\n",
      "        [-0.2345, -1.5153, -0.1951, -0.1611, -0.0989],\n",
      "        [ 0.2793, -0.2709, -0.3643, -0.0088,  0.1430],\n",
      "        [ 0.1801, -2.3441, -0.6238, -0.3838,  0.0601],\n",
      "        [-0.5984,  0.1043,  1.2205,  0.8669,  0.7466]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.6426, 0.5618, 0.4167, 0.8901, 0.9759, 0.5510, 0.0993, 0.4448, 0.0426,\n",
      "        0.9121], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[-0.0237, -0.8489, -0.3580, -0.2837,  0.6630],\n",
      "        [-0.0786,  1.2456, -1.3854, -0.1654, -1.0577],\n",
      "        [-0.4595, -1.1157,  0.7917,  1.1199, -0.7472],\n",
      "        [ 0.8744,  0.1278, -1.2394,  0.2109, -1.4332],\n",
      "        [-0.5988,  2.2654, -0.0954,  0.0928, -0.1329],\n",
      "        [-0.1544,  0.5332,  0.3816,  0.2596,  1.3118],\n",
      "        [-1.6323,  0.3421, -2.3835,  0.0554, -0.0275],\n",
      "        [ 0.1344, -0.8441,  0.0194, -0.1274,  0.0319],\n",
      "        [ 0.0659, -0.0079,  0.2240,  0.5979, -0.1439],\n",
      "        [-0.7174, -1.0515, -1.1510,  0.8066, -2.2776]], grad_fn=<MulBackward0>)\n",
      "torch.Size([10]) tensor([0.2992, 0.1913, 0.3987, 0.1885, 0.8222, 0.9115, 0.0254, 0.3131, 0.6761,\n",
      "        0.0122], grad_fn=<SigmoidBackward>)\n",
      "out: tensor([[ 1.1127, -0.1463,  0.3827, -0.3781,  1.6819]], grad_fn=<MulBackward0>)\n",
      "torch.Size([1]) tensor([0.9342], grad_fn=<SigmoidBackward>)\n",
      "loss: 0.068071  [  950/  981]\n"
     ]
    }
   ],
   "source": [
    "# 训练, 纯演示，数据量比较小，很难收敛\n",
    "learning_rate = 0.0001\n",
    "criterion = nn.BCELoss() #定义损失函数\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) #定义最优化算法\n",
    "\n",
    "def train(dataloader, model, loss_fn, optimizer):\n",
    "    size = len(dataloader.dataset)\n",
    "    model.train()\n",
    "    for batch, (label, in_idx,cxt_idx) in enumerate(dataloader):    \n",
    "        pred = model(in_idx,cxt_idx)\n",
    "        loss = criterion(pred,label)\n",
    "\n",
    "        # Backpropagation\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if batch % 5 == 0:\n",
    "            loss, current = loss.item(), batch * len(label)\n",
    "            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n",
    "            \n",
    "    print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n",
    "\n",
    "train(train_dataloader, model, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8ba96f8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78\n",
      "[78, 16]\n"
     ]
    }
   ],
   "source": [
    "# pytorch 针对自然语言处理的一些封装\n",
    "from torchtext.vocab import vocab\n",
    "from collections import Counter, OrderedDict\n",
    "\n",
    "counter = Counter([token for token in txt])\n",
    "sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[1], reverse=True)\n",
    "ordered_dict = OrderedDict(sorted_by_freq_tuples)\n",
    "\n",
    "\n",
    "v1 = vocab(ordered_dict)\n",
    "print(v1['山'])\n",
    "print(v1.lookup_indices(['山','定']))\n",
    "\n",
    "# torch.txt 0.12支持\n",
    "unk_token = '<unk>'\n",
    "default_index = -1\n",
    "# v2 = vocab(OrderedDict([(token, 1) for token in txt]), specials=[unk_token])\n",
    "# v2.set_default_index(default_index)\n",
    "# print(v2['<unk>']) #prints 0\n",
    "# print(v2['out of vocab']) #prints -1\n",
    "# #make default index same as index of unk_token\n",
    "# v2.set_default_index(v2[unk_token])\n",
    "# v2['out of vocab'] is v2[unk_token] #prints True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ced60b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://wmathor.com/index.php/archives/1569/\n",
    "# n > 8.33logN, N词表大小"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
